Fix for urllib2>=2 (#201)

* Fix for `urllib2>=2`
* Bump version for publishing a new release
* Adding local container for partially replacing httpbin.org usage
* Replacing httpbin.org with mockbin.org from Kong for what needs HTTPS

Author: mindflayer

.github/workflows/main.yml

@@ -20,21 +20,6 @@ jobs:
       matrix:
         python-version: ['3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', 'pypy3.9']
 
-    services:
-      # https://docs.github.com/en/actions/using-containerized-services/about-service-containers
-      redis:
-        # Docker Hub image
-        image: redis
-        # Set health checks to wait until redis has started
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps port 6379 on service container to the host
-          - 6379:6379
-
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
@@ -45,12 +30,24 @@ jobs:
         cache-dependency-path: |
           Pipfile
           setup.py
+    - uses: isbang/compose-action@v1.4.1
+      with:
+        compose-file: "./docker-compose.yml"
+        down-flags: "--remove-orphans"
+        up-flags: "--no-start"
     - name: Install dependencies
       run: |
         make develop
+        make services-up
+    - name: Setup hostname
+      run: |
+        export CONTAINER_ID=$(docker-compose ps -q proxy)
+        export CONTAINER_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $CONTAINER_ID)
+        echo "$CONTAINER_IP httpbin.local" | sudo tee -a /etc/hosts
     - name: Test
       run: |
         make test
+        make services-down
     - name: Push Coveralls
       run: |
         pip install -q coveralls coveralls[yaml]

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
         - id: check-yaml
           args: ['--unsafe']
@@ -18,11 +18,11 @@ repos:
           args: ['--in-place', '--remove-all-unused-imports', '--remove-unused-variable']
 
 -   repo: https://github.com/timothycrosley/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
         - id: isort
 
 -   repo: https://github.com/psf/black
-    rev: 22.6.0
+    rev: 23.3.0
     hooks:
         - id: black

Makefile

@@ -2,15 +2,21 @@
 
 install-dev-requirements:
 	pip install -U pip
-	pip install pipenv pre-commit
+	pip install pipenv
 
 install-test-requirements:
 	pipenv install --dev
 	pipenv run python -c "import pipfile; pf = pipfile.load('Pipfile'); print('\n'.join(package+version if version != '*' else package for package, version in pf.data['default'].items()))" > requirements.txt
 
+services-up:
+	pipenv run docker-compose up -d
+
+services-down:
+	pipenv run docker-compose down --remove-orphans
+
 test-python:
 	@echo "Running Python tests"
-	pipenv run python run_tests.py || exit 1
+	pipenv run wait-for-it --service httpbin.local:443 --service localhost:6379 --timeout 5 -- pipenv run python run_tests.py || exit 1
 	@echo ""
 
 lint-python:
@@ -37,4 +43,5 @@ clean:
 	rm -rf *.egg-info dist/ requirements.txt Pipfile.lock
 	find . -type d -name __pycache__ -exec rm -rf {} \;
 
-.PHONY: clean publish safetest test setup develop lint-python test-python install-test-requirements install-dev-requirements
+.PHONY: clean publish safetest test setup develop lint-python test-python
+.PHONY: services-up services-down install-test-requirements install-dev-requirements

Pipfile

@@ -30,6 +30,8 @@ wheel = "*"
 twine = "*"
 anaconda-client = "*"
 fastapi = "*"
+docker-compose = "*"
+wait-for-it = "*"
 
 [requires]
 python_version = "3"

docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  proxy:
+    container_name: proxy
+    image: shroomlife/docker-https-proxy:latest
+    hostname: "httpbin.local"
+    ports:
+      - "80:80"
+      - "443:443"
+  httpbin:
+    container_name: httpbin.local.proxy
+    image: kennethreitz/httpbin:latest
+  redis:
+    image: redis:latest
+    ports:
+      - "6379:6379"

mocket/__init__.py

@@ -3,4 +3,4 @@
 
 __all__ = ("async_mocketize", "mocketize", "Mocket", "MocketEntry", "Mocketizer")
 
-__version__ = "3.11.0"
+__version__ = "3.11.1"

mocket/mocket.py

@@ -15,7 +15,11 @@
 import urllib3
 from urllib3.connection import match_hostname as urllib3_match_hostname
 from urllib3.util.ssl_ import ssl_wrap_socket as urllib3_ssl_wrap_socket
-from urllib3.util.ssl_ import wrap_socket as urllib3_wrap_socket
+
+try:
+    from urllib3.util.ssl_ import wrap_socket as urllib3_wrap_socket
+except ImportError:
+    urllib3_wrap_socket = None
 
 from .compat import basestring, byte_type, decode_from_bytes, encode_to_bytes, text_type
 from .exceptions import StrictMocketException
@@ -68,6 +72,7 @@ class FakeSetter(int):
         def __set__(self, *args):
             pass
 
+    minimum_version = FakeSetter()
     options = FakeSetter()
     verify_mode = FakeSetter(ssl.CERT_NONE)
 
@@ -164,7 +169,7 @@ class MocketSocket:
     _secure_socket = False
 
     def __init__(
-        self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, *args, **kwargs
+        self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, **kwargs
     ):
         self.true_socket = true_socket(family, type, proto)
         self._buflen = 65536
@@ -380,7 +385,6 @@ def true_sendall(self, data, *args, **kwargs):
 
             # dump the resulting dictionary to a JSON file
             if Mocket.get_truesocket_recording_dir():
-
                 # update the dictionary with request and response lines
                 response_dict["request"] = req
                 response_dict["response"] = hexdump(encoded_response)
@@ -509,12 +513,15 @@ def enable(namespace=None, truesocket_recording_dir=None):
         urllib3.util.ssl_.ssl_wrap_socket = urllib3.util.ssl_.__dict__[
             "ssl_wrap_socket"
         ] = FakeSSLContext.wrap_socket
+        urllib3.util.ssl_wrap_socket = urllib3.util.__dict__[
+            "ssl_wrap_socket"
+        ] = FakeSSLContext.wrap_socket
         urllib3.connection.ssl_wrap_socket = urllib3.connection.__dict__[
             "ssl_wrap_socket"
         ] = FakeSSLContext.wrap_socket
         urllib3.connection.match_hostname = urllib3.connection.__dict__[
             "match_hostname"
-        ] = lambda cert, hostname: None
+        ] = lambda *args: None
         if pyopenssl_override:  # pragma: no cover
             # Take out the pyopenssl version - use the default implementation
             extract_from_urllib3()
@@ -540,6 +547,9 @@ def disable():
         urllib3.util.ssl_.ssl_wrap_socket = urllib3.util.ssl_.__dict__[
             "ssl_wrap_socket"
         ] = true_urllib3_ssl_wrap_socket
+        urllib3.util.ssl_wrap_socket = urllib3.util.__dict__[
+            "ssl_wrap_socket"
+        ] = true_urllib3_ssl_wrap_socket
         urllib3.connection.ssl_wrap_socket = urllib3.connection.__dict__[
             "ssl_wrap_socket"
         ] = true_urllib3_ssl_wrap_socket

mocket/utils.py

@@ -5,7 +5,7 @@
 
 from .compat import decode_from_bytes, encode_to_bytes
 
-SSL_PROTOCOL = ssl.PROTOCOL_SSLv23
+SSL_PROTOCOL = ssl.PROTOCOL_TLSv1_2
 
 
 class MocketSocketCore(io.BytesIO):

tests/main/test_http.py

@@ -28,7 +28,7 @@ def assertEqualHeaders(self, first, second, msg=None):
 class TrueHttpEntryTestCase(HttpTestCase):
     @mocketize
     def test_truesendall(self):
-        url = "http://httpbin.org/ip"
+        url = "http://httpbin.local/ip"
         resp = urlopen(url)
         self.assertEqual(resp.code, 200)
         resp = requests.get(url)
@@ -37,7 +37,7 @@ def test_truesendall(self):
     def test_truesendall_with_recording(self):
         with tempfile.TemporaryDirectory() as temp_dir:
             with Mocketizer(truesocket_recording_dir=temp_dir):
-                url = "http://httpbin.org/ip"
+                url = "http://httpbin.local/ip"
 
                 urlopen(url)
                 requests.get(url)
@@ -54,12 +54,12 @@ def test_truesendall_with_recording(self):
                 with io.open(dump_filename) as f:
                     responses = json.load(f)
 
-                self.assertEqual(len(responses["httpbin.org"]["80"].keys()), 2)
+        self.assertEqual(len(responses["httpbin.local"]["80"].keys()), 2)
 
     def test_truesendall_with_gzip_recording(self):
         with tempfile.TemporaryDirectory() as temp_dir:
             with Mocketizer(truesocket_recording_dir=temp_dir):
-                url = "http://httpbin.org/gzip"
+                url = "http://httpbin.local/gzip"
 
                 requests.get(url)
                 resp = requests.get(url)
@@ -72,12 +72,12 @@ def test_truesendall_with_gzip_recording(self):
                 with io.open(dump_filename) as f:
                     responses = json.load(f)
 
-                assert len(responses["httpbin.org"]["80"].keys()) == 1
+        assert len(responses["httpbin.local"]["80"].keys()) == 1
 
     def test_truesendall_with_chunk_recording(self):
         with tempfile.TemporaryDirectory() as temp_dir:
             with Mocketizer(truesocket_recording_dir=temp_dir):
-                url = "http://httpbin.org/range/70000?chunk_size=65536"
+                url = "http://httpbin.local/range/70000?chunk_size=65536"
 
                 requests.get(url)
                 resp = requests.get(url)
@@ -90,12 +90,14 @@ def test_truesendall_with_chunk_recording(self):
                 with io.open(dump_filename) as f:
                     responses = json.load(f)
 
-                assert len(responses["httpbin.org"]["80"].keys()) == 1
+        assert len(responses["httpbin.local"]["80"].keys()) == 1
 
     @mocketize
     def test_wrongpath_truesendall(self):
-        Entry.register(Entry.GET, "http://httpbin.org/user.agent", Response(status=404))
-        response = urlopen("http://httpbin.org/ip")
+        Entry.register(
+            Entry.GET, "http://httpbin.local/user.agent", Response(status=404)
+        )
+        response = urlopen("http://httpbin.local/ip")
         self.assertEqual(response.code, 200)
 
 
@@ -209,7 +211,7 @@ def test_mockhttp_entry_collect_duplicates(self):
 
     @mocketize
     def test_multipart(self):
-        url = "http://httpbin.org/post"
+        url = "http://httpbin.local/post"
         data = '--xXXxXXyYYzzz\r\nContent-Disposition: form-data; name="content"\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: 68\r\n\r\nAction: comment\nText: Comment with attach\nAttachment: x1.txt, x2.txt\r\n--xXXxXXyYYzzz\r\nContent-Disposition: form-data; name="attachment_2"; filename="x.txt"\r\nContent-Type: text/plain\r\nContent-Length: 4\r\n\r\nbye\n\r\n--xXXxXXyYYzzz\r\nContent-Disposition: form-data; name="attachment_1"; filename="x.txt"\r\nContent-Type: text/plain\r\nContent-Length: 4\r\n\r\nbye\n\r\n--xXXxXXyYYzzz--\r\n'
         headers = {
             "Content-Length": "495",
@@ -233,7 +235,7 @@ def test_multipart(self):
                 "accept-encoding": "identity",
                 "content-length": "495",
                 "content-type": "multipart/form-data; boundary=xXXxXXyYYzzz",
-                "host": "httpbin.org",
+                "host": "httpbin.local",
                 "user-agent": "Mocket",
                 "connection": "keep-alive",
             },
@@ -304,9 +306,11 @@ def test_request_bodies(self):
 
     @mocketize(truesocket_recording_dir=os.path.dirname(__file__))
     def test_truesendall_with_dump_from_recording(self):
-        requests.get("http://httpbin.org/ip", headers={"user-agent": "Fake-User-Agent"})
         requests.get(
-            "http://httpbin.org/gzip", headers={"user-agent": "Fake-User-Agent"}
+            "http://httpbin.local/ip", headers={"user-agent": "Fake-User-Agent"}
+        )
+        requests.get(
+            "http://httpbin.local/gzip", headers={"user-agent": "Fake-User-Agent"}
         )
 
         dump_filename = os.path.join(
@@ -315,7 +319,7 @@ def test_truesendall_with_dump_from_recording(self):
         with io.open(dump_filename) as f:
             responses = json.load(f)
 
-        self.assertEqual(len(responses["httpbin.org"]["80"].keys()), 2)
+        self.assertEqual(len(responses["httpbin.local"]["80"].keys()), 2)
 
     @mocketize
     def test_post_file_object(self):

tests/main/test_http_with_xxhash.py

@@ -6,23 +6,26 @@
 import os
 
 import requests
-from tests.main.test_http import HttpTestCase
 
 from mocket import Mocket, mocketize
+from tests.main.test_http import HttpTestCase
 
 
 class HttpEntryTestCase(HttpTestCase):
-
     @mocketize(truesocket_recording_dir=os.path.dirname(__file__))
     def test_truesendall_with_dump_from_recording(self):
-        requests.get('http://httpbin.org/ip', headers={"user-agent": "Fake-User-Agent"})
-        requests.get('http://httpbin.org/gzip', headers={"user-agent": "Fake-User-Agent"})
+        requests.get(
+            "http://httpbin.local/ip", headers={"user-agent": "Fake-User-Agent"}
+        )
+        requests.get(
+            "http://httpbin.local/gzip", headers={"user-agent": "Fake-User-Agent"}
+        )
 
         dump_filename = os.path.join(
             Mocket.get_truesocket_recording_dir(),
-            Mocket.get_namespace() + '.json',
+            Mocket.get_namespace() + ".json",
         )
         with io.open(dump_filename) as f:
             responses = json.load(f)
 
-        self.assertEqual(len(responses['httpbin.org']['80'].keys()), 2)
+        self.assertEqual(len(responses["httpbin.local"]["80"].keys()), 2)