certusone
/
proxy-model.py
mirror of https://github.com/certusone/proxy-model.py.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

TLS Interception Cert Generation (#362) 

* Use common.pki for interception certificate generation

* Fix tests

* Dont use certificate fields that we dont need, it leads to certificate generation error on Ubuntu

* Prepare for v2.2.0

* npm audit fix
This commit is contained in:
Abhinav Singh 2020-06-09 12:07:00 +05:30 committed by GitHub
parent ab08901239
commit e7aa8a28f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 116 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@ -795,8 +795,7 @@ response from the server. Start `proxy.py` as:
```
> :note: **MacOS users** also need to pass explicit CA file path
> needed for validation of peer certificates. See --ca-file flag.
[![NOTE](https://img.shields.io/static/v1?label=MacOS&message=note&color=yellow)](https://github.com/abhinavsingh/proxy.py#flags) Also provide explicit CA bundle path needed for validation of peer certificates. See `--ca-file` flag.
Verify TLS interception using `curl`
@ -1327,7 +1326,7 @@ usage: pki.py [-h] [--password PASSWORD] [--private-key-path PRIVATE_KEY_PATH]
[--public-key-path PUBLIC_KEY_PATH] [--subject SUBJECT]
action
proxy.py v2.1.2 : PKI Utility
proxy.py v2.2.0 : PKI Utility
positional arguments:
action Valid actions: remove_passphrase, gen_private_key,
@ -1518,7 +1517,7 @@ usage: proxy [-h] [--backlog BACKLOG] [--basic-auth BASIC_AUTH]
[--static-server-dir STATIC_SERVER_DIR] [--threadless]
[--timeout TIMEOUT] [--version]
proxy.py v2.1.2
proxy.py v2.2.0
optional arguments:
-h, --help show this help message and exit

62
dashboard/package-lock.json generated
View File

@ -1382,9 +1382,9 @@
"dev": true
},
"follow-redirects": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.10.0.tgz",
"integrity": "sha512-4eyLK6s6lH32nOvLLwlIOnr9zrL8Sm+OvW4pVTJNoXeGzYIkHVf+pADQi+OJ0E67hiuSLezPVPyBcIZO50TmmQ==",
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.11.0.tgz",
"integrity": "sha512-KZm0V+ll8PfBrKwMzdo5D13b1bur9Iq9Zd/RMmAoQQcl2PxxFml8cxXPaaPYVbV0RjNjq1CU7zIzAOqtUPudmA==",
"dev": true,
"requires": {
"debug": "^3.0.0"
@ -1582,9 +1582,9 @@
}
},
"http-proxy": {
"version": "1.18.0",
"resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.0.tgz",
"integrity": "sha512-84I2iJM/n1d4Hdgc6y2+qY5mDaz2PUVjlg9znE9byl+q0uC3DeByqBGReQu5tpLK0TAqTIXScRUV+dg7+bUPpQ==",
"version": "1.18.1",
"resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz",
"integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==",
"dev": true,
"requires": {
"eventemitter3": "^4.0.0",
@ -1593,19 +1593,19 @@
}
},
"http-server": {
"version": "0.12.1",
"resolved": "https://registry.npmjs.org/http-server/-/http-server-0.12.1.tgz",
"integrity": "sha512-T0jB+7J7GJ2Vo+a4/T7P7SbQ3x2GPDnqRqQXdfEuPuUOmES/9NBxPnDm7dh1HGEeUWqUmLUNtGV63ZC5Uy3tGA==",
"version": "0.12.3",
"resolved": "https://registry.npmjs.org/http-server/-/http-server-0.12.3.tgz",
"integrity": "sha512-be0dKG6pni92bRjq0kvExtj/NrrAd28/8fCXkaI/4piTwQMSDSLMhWyW0NI1V+DBI3aa1HMlQu46/HjVLfmugA==",
"dev": true,
"requires": {
"basic-auth": "^1.0.3",
"colors": "^1.3.3",
"colors": "^1.4.0",
"corser": "^2.0.1",
"ecstatic": "^3.3.2",
"http-proxy": "^1.17.0",
"http-proxy": "^1.18.0",
"minimist": "^1.2.5",
"opener": "^1.5.1",
"optimist": "~0.6.1",
"portfinder": "^1.0.20",
"portfinder": "^1.0.25",
"secure-compare": "3.0.1",
"union": "~0.5.0"
}
@ -2641,30 +2641,6 @@
"pinkie-promise": "^2.0.0"
}
},
"optimist": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz",
"integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=",
"dev": true,
"requires": {
"minimist": "~0.0.1",
"wordwrap": "~0.0.2"
},
"dependencies": {
"minimist": {
"version": "0.0.10",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz",
"integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=",
"dev": true
},
"wordwrap": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz",
"integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=",
"dev": true
}
}
},
"optionator": {
"version": "0.8.2",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz",
@ -2832,9 +2808,9 @@
"dev": true
},
"portfinder": {
"version": "1.0.25",
"resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.25.tgz",
"integrity": "sha512-6ElJnHBbxVA1XSLgBp7G1FiCkQdlqGzuF7DswL5tcea+E8UpuvPU7beVAjjRwCioTS9ZluNbu+ZyRvgTsmqEBg==",
"version": "1.0.26",
"resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.26.tgz",
"integrity": "sha512-Xi7mKxJHHMI3rIUrnm/jjUgwhbYMkp/XKEcZX3aG4BrumLpq3nmoQMX+ClYnDZnZ/New7IatC1no5RX0zo1vXQ==",
"dev": true,
"requires": {
"async": "^2.6.2",
@ -2873,9 +2849,9 @@
"dev": true
},
"qs": {
"version": "6.9.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.9.2.tgz",
"integrity": "sha512-2eQ6zajpK7HwqrY1rRtGw5IZvjgtELXzJECaEDuzDFo2jjnIXpJSimzd4qflWZq6bLLi+Zgfj5eDrAzl/lptyg==",
"version": "6.9.4",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz",
"integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==",
"dev": true
},
"read-pkg": {

2
dashboard/package.json
View File

@ -37,7 +37,7 @@
"eslint-plugin-node": "^10.0.0",
"eslint-plugin-promise": "^4.2.1",
"eslint-plugin-standard": "^4.0.1",
"http-server": "^0.12.1",
"http-server": "^0.12.3",
"jasmine": "^3.5.0",
"jasmine-ts": "^0.3.0",
"jquery": "^3.5.0",

2
helper/homebrew/stable/proxy.rb
View File

@ -5,7 +5,7 @@ class Proxy < Formula
Network monitoring, controls & Application development, testing, debugging."
homepage "https://github.com/abhinavsingh/proxy.py"
url "https://github.com/abhinavsingh/proxy.py/archive/master.zip"
version "2.0.0"
version "2.1.2"
depends_on "python"

2
proxy/common/version.py
View File

@ -8,5 +8,5 @@
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
VERSION = (2, 1, 2)
VERSION = (2, 2, 0)
__version__ = '.'.join(map(str, VERSION[0:3]))

74
proxy/http/proxy/server.py
View File

@ -8,14 +8,13 @@
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
import logging
import threading
import subprocess
import os
import ssl
import socket
import time
import errno
import logging
from typing import Optional, List, Union, Dict, cast, Any, Tuple
from .plugin import HttpProxyBasePlugin
@ -28,6 +27,7 @@ from ..methods import httpMethods
from ...common.types import HasFileno
from ...common.constants import PROXY_AGENT_HEADER_VALUE
from ...common.utils import build_http_response, text_
from ...common.pki import gen_public_key, gen_csr, sign_csr
from ...core.event import eventNames
from ...core.connection import TcpServerConnection, TcpConnectionUninitializedException
@ -279,7 +279,8 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin):
'BrokenPipeError when wrapping client')
return True
except OSError as e:
logger.exception('OSError when wrapping client', exc_info=e)
logger.exception(
'OSError when wrapping client', exc_info=e)
return True
# Update all plugin connection reference
for plugin in self.plugins.values():
@ -342,6 +343,57 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin):
self.response.total_size,
connection_time_ms))
def gen_ca_signed_certificate(self, cert_file_path: str) -> None:
'''CA signing key (default) is used for generating a public key
for common_name, if one already doesn't exist. Using generated
public key a CSR request is generated, which is then signed by
CA key and secret. Again this process only happen if signed
certificate doesn't already exist.
returns signed certificate path.'''
assert(self.request.host and self.flags.ca_cert_dir and self.flags.ca_signing_key_file and
self.flags.ca_key_file and self.flags.ca_cert_file)
public_key_path = os.path.join(self.flags.ca_cert_dir,
'{0}.{1}'.format(text_(self.request.host), 'pub'))
private_key_path = self.flags.ca_signing_key_file
private_key_password = ''
subject = '/CN={0}'.format(text_(self.request.host))
alt_subj_names = [text_(self.request.host), ]
validity_in_days = 365 * 2
timeout = 10
# Generate a public key for the common name
if not os.path.isfile(public_key_path):
logger.debug('Generating public key %s', public_key_path)
resp = gen_public_key(public_key_path=public_key_path, private_key_path=private_key_path,
private_key_password=private_key_password, subject=subject, alt_subj_names=alt_subj_names,
validity_in_days=validity_in_days, timeout=timeout)
assert(resp is True)
csr_path = os.path.join(self.flags.ca_cert_dir,
'{0}.{1}'.format(text_(self.request.host), 'csr'))
# Generate a CSR request for this common name
if not os.path.isfile(csr_path):
logger.debug('Generating CSR %s', csr_path)
resp = gen_csr(csr_path=csr_path, key_path=private_key_path, password=private_key_password,
crt_path=public_key_path, timeout=timeout)
assert(resp is True)
ca_key_path = self.flags.ca_key_file
ca_key_password = ''
ca_crt_path = self.flags.ca_cert_file
serial = self.uid.int
# Sign generated CSR
if not os.path.isfile(cert_file_path):
logger.debug('Signing CSR %s', cert_file_path)
resp = sign_csr(csr_path=csr_path, crt_path=cert_file_path, ca_key_path=ca_key_path,
ca_key_password=ca_key_password, ca_crt_path=ca_crt_path,
serial=str(serial), alt_subj_names=alt_subj_names,
validity_in_days=validity_in_days, timeout=timeout)
assert(resp is True)
@staticmethod
def generated_cert_file_path(ca_cert_dir: str, host: str) -> str:
return os.path.join(ca_cert_dir, '%s.pem' % host)
@ -359,21 +411,7 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin):
self.flags.ca_cert_dir, text_(self.request.host))
with self.lock:
if not os.path.isfile(cert_file_path):
logger.debug('Generating certificates %s', cert_file_path)
# TODO: Parse subject from certificate
# Currently we only set CN= field for generated certificates.
gen_cert = subprocess.Popen(
['openssl', 'req', '-new', '-key', self.flags.ca_signing_key_file, '-subj',
f'/C=/ST=/L=/O=/OU=/CN={ text_(self.request.host) }'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sign_cert = subprocess.Popen(
['openssl', 'x509', '-req', '-days', '365', '-CA', self.flags.ca_cert_file, '-CAkey',
self.flags.ca_key_file, '-set_serial', str(self.uid.int), '-out', cert_file_path],
stdin=gen_cert.stdout,
stderr=subprocess.PIPE)
# TODO: Ensure sign_cert success.
sign_cert.communicate(timeout=10)
self.gen_ca_signed_certificate(cert_file_path)
return cert_file_path
def wrap_server(self) -> None:

2
setup.py
View File

@ -10,7 +10,7 @@
"""
from setuptools import setup, find_packages
VERSION = (2, 1, 2)
VERSION = (2, 2, 0)
__version__ = '.'.join(map(str, VERSION[0:3]))
__description__ = '''⚡⚡⚡Fast, Lightweight, Pluggable, TLS interception capable proxy server
focused on Network monitoring, controls & Application development, testing, debugging.'''

22
tests/http/test_http_proxy_tls_interception.py
View File

@ -30,14 +30,18 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
@mock.patch('ssl.wrap_socket')
@mock.patch('ssl.create_default_context')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
@mock.patch('subprocess.Popen')
@mock.patch('proxy.http.proxy.server.gen_public_key')
@mock.patch('proxy.http.proxy.server.gen_csr')
@mock.patch('proxy.http.proxy.server.sign_csr')
@mock.patch('selectors.DefaultSelector')
@mock.patch('socket.fromfd')
def test_e2e(
self,
mock_fromfd: mock.Mock,
mock_selector: mock.Mock,
mock_popen: mock.Mock,
mock_sign_csr: mock.Mock,
mock_gen_csr: mock.Mock,
mock_gen_public_key: mock.Mock,
mock_server_conn: mock.Mock,
mock_ssl_context: mock.Mock,
mock_ssl_wrap: mock.Mock) -> None:
@ -46,11 +50,17 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
self.mock_fromfd = mock_fromfd
self.mock_selector = mock_selector
self.mock_popen = mock_popen
self.mock_sign_csr = mock_sign_csr
self.mock_gen_csr = mock_gen_csr
self.mock_gen_public_key = mock_gen_public_key
self.mock_server_conn = mock_server_conn
self.mock_ssl_context = mock_ssl_context
self.mock_ssl_wrap = mock_ssl_wrap
self.mock_sign_csr.return_value = True
self.mock_gen_csr.return_value = True
self.mock_gen_public_key.return_value = True
ssl_connection = mock.MagicMock(spec=ssl.SSLSocket)
self.mock_ssl_context.return_value.wrap_socket.return_value = ssl_connection
self.mock_ssl_wrap.return_value = mock.MagicMock(spec=ssl.SSLSocket)
@ -118,6 +128,7 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
fd=self._conn.fileno,
events=selectors.EVENT_READ,
data=None), selectors.EVENT_READ)], ]
self.protocol_handler.run_once()
# Assert our mocked plugins invocations
@ -142,8 +153,9 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
self.assertEqual(plain_connection.setblocking.call_count, 2)
self.mock_ssl_context.return_value.wrap_socket.assert_called_with(
plain_connection, server_hostname=host)
# TODO: Assert Popen arguments, piping, success condition
self.assertEqual(self.mock_popen.call_count, 2)
self.assertEqual(self.mock_sign_csr.call_count, 1)
self.assertEqual(self.mock_gen_csr.call_count, 1)
self.assertEqual(self.mock_gen_public_key.call_count, 1)
self.assertEqual(ssl_connection.setblocking.call_count, 1)
self.assertEqual(
self.mock_server_conn.return_value._conn,

21
tests/plugin/test_http_proxy_plugins_with_tls_interception.py
View File

@ -33,23 +33,33 @@ class TestHttpProxyPluginExamplesWithTlsInterception(unittest.TestCase):
@mock.patch('ssl.wrap_socket')
@mock.patch('ssl.create_default_context')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
@mock.patch('subprocess.Popen')
@mock.patch('proxy.http.proxy.server.gen_public_key')
@mock.patch('proxy.http.proxy.server.gen_csr')
@mock.patch('proxy.http.proxy.server.sign_csr')
@mock.patch('selectors.DefaultSelector')
@mock.patch('socket.fromfd')
def setUp(self,
mock_fromfd: mock.Mock,
mock_selector: mock.Mock,
mock_popen: mock.Mock,
mock_sign_csr: mock.Mock,
mock_gen_csr: mock.Mock,
mock_gen_public_key: mock.Mock,
mock_server_conn: mock.Mock,
mock_ssl_context: mock.Mock,
mock_ssl_wrap: mock.Mock) -> None:
self.mock_fromfd = mock_fromfd
self.mock_selector = mock_selector
self.mock_popen = mock_popen
self.mock_sign_csr = mock_sign_csr
self.mock_gen_csr = mock_gen_csr
self.mock_gen_public_key = mock_gen_public_key
self.mock_server_conn = mock_server_conn
self.mock_ssl_context = mock_ssl_context
self.mock_ssl_wrap = mock_ssl_wrap
self.mock_sign_csr.return_value = True
self.mock_gen_csr.return_value = True
self.mock_gen_public_key.return_value = True
self.fileno = 10
self._addr = ('127.0.0.1', 54382)
self.flags = Flags(
@ -126,7 +136,10 @@ class TestHttpProxyPluginExamplesWithTlsInterception(unittest.TestCase):
)
self.protocol_handler.run_once()
self.mock_popen.assert_called()
self.assertEqual(self.mock_sign_csr.call_count, 1)
self.assertEqual(self.mock_gen_csr.call_count, 1)
self.assertEqual(self.mock_gen_public_key.call_count, 1)
self.mock_server_conn.assert_called_once_with('uni.corn', 443)
self.server.connect.assert_called()
self.assertEqual(