From 22936cd8ed0f0d5b0c3ece75752e2877f46af455 Mon Sep 17 00:00:00 2001

From: Patrick Uiterwijk <puiterwijk@redhat.com>

Date: Wed, 21 Sep 2016 13:48:07 +0000

Subject: [PATCH 1/3] Rewrite http.py to use python-requests

This will work around an issue when pycurl is used in the same context as

rpm is, as used by koji-containerbuild.

Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>

---

 osbs/http.py | 290 +++++++++--------------------------------------------------

 1 file changed, 42 insertions(+), 248 deletions(-)

diff --git a/osbs/http.py b/osbs/http.py

index 9955c55..5f39141 100644

--- a/osbs/http.py

+++ b/osbs/http.py

@@ -24,77 +24,17 @@

 import logging

 from io import BytesIO

-import pycurl

-

 from osbs.exceptions import OsbsException, OsbsNetworkException, OsbsResponseException

+import requests

 try:

-    # py2

-    import httplib

+    from requests_kerberos import HTTPKerberosAuth

 except ImportError:

-    # py3

-    import http.client as httplib

+    HTTPKerberosAuth = None

-logger = logging.getLogger(__name__)

-SELECT_TIMEOUT = 9999

-PYCURL_NETWORK_CODES = [pycurl.E_BAD_CONTENT_ENCODING,

-                        pycurl.E_BAD_DOWNLOAD_RESUME,

-                        pycurl.E_CONV_FAILED,

-                        pycurl.E_CONV_REQD,

-                        pycurl.E_COULDNT_CONNECT,

-                        pycurl.E_COULDNT_RESOLVE_HOST,

-                        pycurl.E_COULDNT_RESOLVE_PROXY,

-                        pycurl.E_FILESIZE_EXCEEDED,

-                        pycurl.E_HTTP_POST_ERROR,

-                        pycurl.E_HTTP_RANGE_ERROR,

-                        pycurl.E_HTTP_RETURNED_ERROR,

-                        pycurl.E_LOGIN_DENIED,

-                        # old pycurl: E_OPERATION_TIMEOUTED, new pycurl: E_OPERATION_TIMEDOUT

-                        getattr(pycurl, "E_OPERATION_TIMEDOUT", "E_OPERATION_TIMEOUTED"),

-                        pycurl.E_PARTIAL_FILE,

-                        pycurl.E_READ_ERROR,

-                        pycurl.E_RECV_ERROR,

-                        pycurl.E_REMOTE_FILE_NOT_FOUND,

-                        pycurl.E_SEND_ERROR,

-                        pycurl.E_SSL_CACERT,

-                        pycurl.E_SSL_CERTPROBLEM,

-                        pycurl.E_SSL_CIPHER,

-                        pycurl.E_SSL_CONNECT_ERROR,

-                        pycurl.E_SSL_PEER_CERTIFICATE,

-                        pycurl.E_SSL_SHUTDOWN_FAILED,

-                        pycurl.E_TOO_MANY_REDIRECTS,

-                        pycurl.E_UNSUPPORTED_PROTOCOL,

-                        pycurl.E_WRITE_ERROR]

-

-PYCURL_NETWORK_CODES = [x for x in PYCURL_NETWORK_CODES if x is not None]

-

-

-def parse_headers(all_headers):

-    # all_headers contains headers from all responses - even without FOLLOWLOCATION there

-    # might be multiple sets of headers due to 401 Unauthorized. We only care about the last

-    # response.

-    try:

-        raw_headers = all_headers.split(b"\r\n\r\n")[-2]

-    except IndexError:

-        logger.warning('Incorrectly terminated http headers')

-        raw_headers = all_headers

-

-    logger.debug("raw headers: " + repr(raw_headers))

-

-    # http://stackoverflow.com/questions/24728088/python-parse-http-response-string/24729316#24729316

-    class FakeSocket(object):

-        def __init__(self, response_str):

-            self._file = BytesIO(response_str)

-

-        def makefile(self, *args, **kwargs):

-            return self._file

-

-    response = httplib.HTTPResponse(FakeSocket(raw_headers))

-    response.begin()

-    header_list = [(k.lower(), v) for (k, v) in response.getheaders()]

-    return dict(header_list)

+logger = logging.getLogger(__name__)

 class HttpSession(object):

@@ -120,22 +60,9 @@ def request(self, url, *args, **kwargs):

                 return stream

             with stream as s:

-                # joining at once is much faster than doing += in a loop

-                all_chunks = list(s.iter_chunks())

-                content = ''.join(all_chunks)

+                content = s.req.content

                 return HttpResponse(s.status_code, s.headers, content)

-        except pycurl.error as ex:

-            code = ex.args[0]

-            try:

-                message = ex.args[1]

-            except IndexError:

-                # happened on rhel 6

-                message = ""

-            if code in PYCURL_NETWORK_CODES:

-                raise OsbsNetworkException(url, message, code, *ex.args[2:],

-                                           cause=ex,

-                                           traceback=sys.exc_info()[2])

-

+        except Exception as ex:

             raise OsbsException(cause=ex, traceback=sys.exc_info()[2])

@@ -160,203 +87,68 @@ def __init__(self, url, method, data=None, kerberos_auth=False,

         self.status_code = 0

         self.headers = None

-        self.response_buffer = BytesIO()

-        self.headers_buffer = BytesIO()

-        self.response_decoder = None

         self.url = url

         headers = headers or {}

         method = method.lower()

-        self.c = pycurl.Curl()

-        self.curl_multi = pycurl.CurlMulti()

-

-        if method == 'post':

-            self.c.setopt(pycurl.POST, 1)

-            headers["Expect"] = ""  # openshift can't handle Expect

-        elif method == 'get':

-            self.c.setopt(pycurl.HTTPGET, 1)

-        elif method == 'put':

-            # self.c.setopt(pycurl.PUT, 1)

-            self.c.setopt(pycurl.CUSTOMREQUEST, b"PUT")

-            headers["Expect"] = ""

-        elif method == 'delete':

-            self.c.setopt(pycurl.CUSTOMREQUEST, b"DELETE")

-        else:

+        if method not in ['post', 'get', 'put', 'delete']:

             raise RuntimeError("Unsupported method '%s' for curl call!" % method)

-        self.c.setopt(pycurl.COOKIEFILE, b'')

-        self.c.setopt(pycurl.URL, str(url))

-        self.c.setopt(pycurl.WRITEFUNCTION, self.response_buffer.write)

-        self.c.setopt(pycurl.HEADERFUNCTION, self.headers_buffer.write)

-        self.c.setopt(pycurl.DEBUGFUNCTION, self._curl_debug)

-        self.c.setopt(pycurl.SSL_VERIFYPEER, 1 if verify_ssl else 0)

-        self.c.setopt(pycurl.SSL_VERIFYHOST, 2 if verify_ssl else 0)

-        if ca:

-            logger.info("Setting CAINFO to %r", ca)

-            self.c.setopt(pycurl.CAINFO, ca)

-

-        self.c.setopt(pycurl.VERBOSE, 1 if verbose else 0)

+        args = {}

+

+        if method in ['post', 'put']:

+            headers['Expect'] = ''

+

+        if not verify_ssl:

+            args['verify'] = False

+        else:

+            if ca:

+                args['verify'] = ca

+            else:

+                args['verify'] = True

+

         if username and password:

-            username = username.encode('utf-8')

-            password = password.encode('utf-8')

-            self.c.setopt(pycurl.USERPWD, username + b":" + password)

+            args['auth'] = (username, password)

         if client_cert and client_key:

-            self.c.setopt(pycurl.SSLCERTTYPE, "PEM")

-            self.c.setopt(pycurl.SSLKEYTYPE, "PEM")

-            self.c.setopt(pycurl.SSLCERT, client_cert)

-            self.c.setopt(pycurl.SSLKEY, client_key)

+            args['cert'] = (client_cert, client_key)

         if data:

-            # curl sets the method to post if one sets any POSTFIELDS (even '')

-            self.c.setopt(pycurl.POSTFIELDS, data)

+            args['data'] = data

         if use_json:

-            headers['Content-Type'] = b'application/json'

+            headers['Content-Type'] = 'application/json'

-        if allow_redirects:

-            self.c.setopt(pycurl.FOLLOWLOCATION, 1)

+        args['allow_redirects'] = allow_redirects

         if kerberos_auth:

-            self.c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)

-            self.c.setopt(pycurl.USERPWD, b':')

+            if not HTTPKerberosAuth:

+                raise RuntimeError('Kerberos auth unavailable')

+            args['auth'] = HTTPKerberosAuth()

         if stream:

-            headers['Cache-Control'] = b'no-cache'

-

-        if headers:

-            header_list = []

-            for header_key, header_value in headers.items():

-                header_list.append(str("%s: %s" % (header_key, header_value)))

-            self.c.setopt(pycurl.HTTPHEADER, header_list)

-

-        self.curl_multi.add_handle(self.c)

-

-        # Send request and read all headers. We have all headers once we receive some data or once

-        # the response ends.

-        # NOTE: HTTP response in chunked encoding can contain additional headers ("trailers") in the

-        # last chunk. This is not handled here.

-        while not (self.finished or self._any_data_received()):

-            self._select()

-            self._perform()

-

-        self.headers = parse_headers(self.headers_buffer.getvalue())

-        self.status_code = self.c.getinfo(pycurl.HTTP_CODE)

-        self.response_decoder = codecs.getincrementaldecoder(self.encoding)()

-

-    def _perform(self):

-        while True:

-            ret, num_handles = self.curl_multi.perform()

-            if ret != pycurl.E_CALL_MULTI_PERFORM:

-                # see curl_multi_perform manpage

-                break

-

-        num_q, _, err_list = self.curl_multi.info_read()

-        if num_q != 0:

-            logger.warning("CurlMulti.info_read() has %s remaining messages", num_q)

-

-        if err_list:

-            err_obj = err_list[0]

-

-            # For anything but the connection being closed, raise

-            if err_obj[1] != pycurl.E_PARTIAL_FILE:

-                raise OsbsNetworkException(self.url, err_obj[2], err_obj[1])

-

-        self.finished = (num_handles == 0)

-

-    def _select(self):

-        sel = self.curl_multi.select(SELECT_TIMEOUT)

-        if sel == -1:

-            raise OsbsException("CurlMulti.select() timed out")

-        elif sel == 0:

-            # sel==0 means curl_multi_fdset returned -1

-            # manual page suggests sleeping >100ms when this happens:(

-            time.sleep(0.1)

-

-    def _any_data_received(self):

-        return self.response_buffer.tell() != 0

+            args['stream'] = True

+

+        args['headers'] = headers

+        self.req = requests.request(method, url, **args)

+

+        self.headers = self.req.headers

+        self.status_code = self.req.status_code

     def _get_received_data(self):

-        result = self.response_buffer.getvalue()

-        self.response_buffer.truncate(0)

-        self.response_buffer.seek(0)

-        return self.response_decoder.decode(result, final=self.finished)

+        return self.req.content

     def iter_chunks(self):

-        while True:

-            self._perform()

-            if self._any_data_received():

-                yield self._get_received_data()

-            if self.finished:

-                break

-            self._select()

-

-        logger.debug("end of the stream")

-        self.close()

+        return self.req.iter_content(None)

     def iter_lines(self):

-        chunks = self.iter_chunks()

-        return self._split_lines_from_chunks(chunks)

-

-    @staticmethod

-    def _split_lines_from_chunks(chunks):

-        # same behaviour as requests' Response.iter_lines(...)

-

-        pending = None

-        for chunk in chunks:

-

-            if pending is not None:

-                chunk = pending + chunk

-            lines = chunk.splitlines()

-

-            if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:

-                pending = lines.pop()

-            else:

-                pending = None

-

-            for line in lines:

-                yield line

-

-        if pending is not None:

-            yield pending

-

-    @staticmethod

-    def _curl_debug(debug_type, debug_msg):

-        try:

-            logger_name = {

-                pycurl.INFOTYPE_TEXT: 'curl',

-                pycurl.INFOTYPE_HEADER_IN: 'in',

-                pycurl.INFOTYPE_HEADER_OUT: 'out'

-            }[debug_type]

-        except KeyError:

-            return

-

-        curl_logger = logging.getLogger(__name__ + '.' + logger_name)

-        for line in debug_msg.splitlines():

-            if not line:

-                continue

-            curl_logger.debug(line)

-

-    @property

-    def encoding(self):

-        encoding = None

-        if 'content-type' in self.headers:

-            content_type = self.headers['content-type'].lower()

-            match = re.search(r'charset=(\S+)', content_type)

-            if match:

-                encoding = match.group(1)

-        if encoding is None:

-            encoding = 'utf-8'  # assume utf-8

-

-        return encoding

+        return self.req.iter_lines()

     def close(self):

         if not self.closed:

             logger.debug("cleaning up")

-            self.curl_multi.remove_handle(self.c)

-            self.c.close()

-            self.curl_multi.close()

+            del self.req

         self.closed = True

     def __del__(self):

@@ -376,7 +168,9 @@ def __init__(self, status_code, headers, content):

         self.content = content

     def json(self, check=True):

-        if check and self.status_code not in (0, httplib.OK, httplib.CREATED):

+        if check and self.status_code not in (0, requests.codes.OK, requests.codes.CREATED):

             raise OsbsResponseException(self.content, self.status_code)

+        if isinstance(self.content, bytes):

+            self.content = self.content.decode('utf-8')

         return json.loads(self.content)

From b626346936a0cd12cdaed028152311f70afcdc41 Mon Sep 17 00:00:00 2001

From: Patrick Uiterwijk <puiterwijk@redhat.com>

Date: Wed, 21 Sep 2016 14:13:25 +0000

Subject: [PATCH 2/3] Fix tests for porting of http client to requests

Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>

---

 tests/test_http.py | 58 +-----------------------------------------------------

 1 file changed, 1 insertion(+), 57 deletions(-)

diff --git a/tests/test_http.py b/tests/test_http.py

index 674f9eb..30d2f14 100644

--- a/tests/test_http.py

+++ b/tests/test_http.py

@@ -12,7 +12,7 @@

 import pytest

 import osbs.http as osbs_http

-from osbs.http import parse_headers, HttpSession, HttpStream

+from osbs.http import HttpSession, HttpStream

 from osbs.exceptions import OsbsNetworkException

 from tests.fake_api import Connection, ResponseMapping

@@ -34,22 +34,6 @@ def has_connection():

         return False

-class TestParseHeaders(object):

-    def test_parse_headers(self):

-        conn = Connection("0.5.4")

-        rm = ResponseMapping("0.5.4", lookup=conn.get_definition_for)

-

-        key, value = conn.get_definition_for("/oauth/authorize")

-        file_name = value["get"]["file"]

-        raw_headers = rm.get_response_content(file_name)

-

-        headers = parse_headers(raw_headers)

-

-        assert headers is not None

-        assert len(headers.items()) > 0

-        assert headers["location"]

-

-

 @pytest.mark.skipif(not has_connection(),

                     reason="requires internet connection")

 class TestHttpSession(object):

@@ -141,43 +125,3 @@ def test_raise(self, s):

             with s.get("http://httpbin.org/stream/3", stream=True) as s:

                 raise RuntimeError("hi")

         assert s.closed

-

-

-class TestHttpStream(object):

-    @pytest.mark.parametrize('chunks,expected_content', [

-        ([b'foo', b'', b'bar', b'baz'], u'foobarbaz'),

-        ([b'a', b'b', b'\xc4', b'\x8d', b'x'], u'ab\u010dx'),

-        ([b'\xe2', b'\x8a', b'\x86'], u'\u2286'),

-        ([b'\xe2\x8a', b'\x86'], u'\u2286'),

-        ([b'\xe2', b'\x8a\x86'], u'\u2286'),

-        ([b'aaaa', b'\xe2\x8a', b'\x86'], u'aaaa\u2286'),

-        ([b'aaaa\xe2\x8a', b'\x86'], u'aaaa\u2286'),

-        ([b'\xe2\x8a', b'\x86ffff'], u'\u2286ffff'),

-    ])

-    def test_http_multibyte_decoding(self, chunks, expected_content):

-        class Whatever(object):

-            def __getattr__(self, name):

-                return self

-

-            def __call__(self, *args, **kwargs):

-                return self

-        flexmock(pycurl).should_receive('Curl').and_return(Whatever())

-        flexmock(pycurl).should_receive('CurlMulti').and_return(Whatever())

-        (flexmock(osbs_http).should_receive('parse_headers')

-                            .and_return({'content-type': 'application/json; charset=utf-8'}))

-        flexmock(HttpStream, _select=lambda: None)

-

-        def mock_perform(self):

-            if chunks:

-                self.response_buffer.write(chunks.pop(0))

-            else:

-                self.finished = True

-

-        try:

-            orig_perform = HttpStream._perform

-            HttpStream._perform = mock_perform

-

-            r = HttpSession(verbose=True).get('http://')

-            assert r.content == expected_content

-        finally:

-            HttpStream._perform = orig_perform

From cdd2061d6397e159bb32413b39e538be7685b505 Mon Sep 17 00:00:00 2001

From: Patrick Uiterwijk <puiterwijk@redhat.com>

Date: Wed, 21 Sep 2016 14:27:16 +0000

Subject: [PATCH 3/3] Update spec file to include python-requests

Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>

---

 osbs-client.spec | 12 ++++++++----

 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/osbs-client.spec b/osbs-client.spec

index 298aadf..c0b3cd5 100644

--- a/osbs-client.spec

+++ b/osbs-client.spec

@@ -60,7 +60,8 @@ BuildRequires:  python-pytest-capturelog

 BuildRequires:  python-flexmock

 BuildRequires:  python-six

 BuildRequires:  python-dockerfile-parse

-BuildRequires:  python-pycurl

+BuildRequires:  python-requests

+BuildRequires:  python-requests-kerberos

 %endif # with_check

 %if 0%{?with_python3}

@@ -73,7 +74,8 @@ BuildRequires:  python3-pytest-capturelog

 BuildRequires:  python3-flexmock

 BuildRequires:  python3-six

 BuildRequires:  python3-dockerfile-parse

-BuildRequires:  python3-pycurl

+BuildRequires:  python3-requests

+BuildRequires:  python3-requests-kerberos

 %endif # with_check

 %endif # with_python3

@@ -91,7 +93,8 @@ Summary:        Python 2 module for OpenShift Build Service

 Group:          Development/Tools

 License:        BSD

 Requires:       python-dockerfile-parse

-Requires:       python-pycurl

+Requires:       python-requests

+Requires:       python-requests-kerberos

 Requires:       python-setuptools

 Requires:       krb5-workstation

 %if 0%{?rhel} && 0%{?rhel} <= 6

@@ -113,7 +116,8 @@ Summary:        Python 3 module for OpenShift Build Service

 Group:          Development/Tools

 License:        BSD

 Requires:       python3-dockerfile-parse

-Requires:       python3-pycurl

+Requires:       python3-requests

+Requires:       python3-requests-kerberos

 Requires:       python3-dateutil

 Requires:       python3-setuptools

 Requires:       krb5-workstation