带有 libcurl 的线程安全代理

thread-safety proxy with libcurl

本文关键字:安全 代理 线程 libcurl 带有      更新时间:2023-10-16
    #pragma once
    #ifndef __CURL_CURL_H
    #include "curl.h"
    #endif
    #ifndef __CURL_EASY_H
    #include "easy.h"
    #endif
    #include <stdint.h>
    #include <memory>
    #include <string>
    namespace CommUnit
    {
        enum ERR_PROXY
        {
            ERR_CURL_INIT_FAILED = 0xA0,
            ERR_SET_PROXY_FAILED = 0xA1,
        };

        class MyProxy
        {
        public:
            static MyProxy & GetInstance()  //Meyers' Singlton
            {
                static MyProxy ProxySigleton;
                return ProxySigleton;
            }
        public:
            /*
            * @bref:Get request
            * @param[in] sUrl:Access URL
            * @param[in] sProxyIp:Proxy IP
            * @param[in] uProxyPort:Proxy Port
            * @param[in] uTimeOut:Time out
            * @param[in] isSSL:HTTPS true,else false
            * @param[out] sRetContent:Return the URL content
            */
            uint32_t Get(const std::string &sUrl,
                const std::string& sProxyIp,
                uint32_t uProxyPort,
                uint32_t uTimeOut,
                bool isSSL,
                std::string &sRetContent);
        private:
            MyProxy();                              //Constructor hidden
            MyProxy(MyProxy const &);               //Copy-Constructor hidden
            MyProxy & operator= (MyProxy const &);  //Assign operator hidden
            ~MyProxy();                             //Destructor hidden
            inline void _setCurlopt(CURL *pCurl,
                const std::string &sUrl,
                std::string &sWriterData,
                const uint32_t uTimeOut,
                bool isSSL);
            //Callback function, write data to writerData
            static int Writer(char *data,
                uint32_t size,
                uint32_t nmemb,
                std::string *writerData);

        private:
            std::string m_sErrMsg;
            static char s_ErrBuffer[CURL_ERROR_SIZE];
            static const uint32_t m_MAXBUF = 2 * 1024 * 1024 - 128; 
        };
    }

//////////////////////////////////////////////////////////////////////////
#include <iostream>
#include <string>
#include "MyProxy.h"
#include "Log.h"
#include <curl.h>
using namespace CommUnit;
char MyProxy::s_ErrBuffer[CURL_ERROR_SIZE] = { 0 };

MyProxy::MyProxy(void)
{
    CURLcode oCURLcode = curl_global_init(CURL_GLOBAL_ALL);
    if (oCURLcode != CURLE_OK)
    {
        Log("ERR: %s curl_init failed!", __func__);
    }
}
MyProxy::~MyProxy(void)
{
    curl_global_cleanup();
}

uint32_t MyProxy::Get(const std::string &sUrl,
    const std::string& sProxyIp,
    uint32_t uProxyPort,
    uint32_t uTimeOut,
    bool isSSL,
    std::string &sRetContent)
{
    sRetContent.clear();
    CURL *pCurl = curl_easy_init();
    CURLcode oCURLcode;
    if (nullptr == pCurl)
    {
        Log("ERR: %s curl_easy_init failed!", __func__);
        return ERR_CURL_INIT_FAILED;
    }
    _setCurlopt(pCurl, sUrl, sRetContent, uTimeOut, isSSL);
    if (0 == sProxyIp.length()|| 0 == uProxyPort)
    {
        Log("ERR: %s SetProxy: ProxyIp [%s], ProxyPort[%u] failed",__func__, sProxyIp.c_str(), uProxyPort);
        return ERR_SET_PROXY_FAILED;
    }
    Log("INFO: %s SetProxy: ProxyIp [%s], ProxyPort[%u] failed", __func__, sProxyIp.c_str(), uProxyPort);
    curl_easy_setopt(pCurl, CURLOPT_PROXY, sProxyIp.c_str());
    curl_easy_setopt(pCurl, CURLOPT_PROXYPORT, uProxyPort);
    int iTimes = 0;
    while (true)
    {
        oCURLcode = curl_easy_perform(pCurl);
        if (oCURLcode != CURLE_OK && ++iTimes < 3)
            usleep(5);
        else
            break;
    }
    if (oCURLcode != CURLE_OK)
    {
        Log("ERR: %s curl_easy_perform failed!", __func__);
    }
    curl_easy_cleanup(pCurl);
    return oCURLcode;
}
void MyProxy::_setCurlopt(CURL *pCurl,
    const std::string &sUrl,
    std::string &sWriterData,
    const uint32_t uTimeOut,
    bool isSSL)
{
    curl_easy_setopt(pCurl, CURLOPT_ERRORBUFFER, s_ErrBuffer);
    curl_easy_setopt(pCurl, CURLOPT_FOLLOWLOCATION, 1);
    curl_easy_setopt(pCurl, CURLOPT_URL, sUrl.c_str());
    curl_easy_setopt(pCurl, CURLOPT_TIMEOUT, uTimeOut);
    Log("INFO: %s Set Url:[%s],TimeOut:[%d]", __func__, sUrl.c_str(), uTimeOut);
    curl_easy_setopt(pCurl, CURLOPT_WRITEFUNCTION, MyProxy::Writer);
    curl_easy_setopt(pCurl, CURLOPT_WRITEDATA, &sWriterData);
    //Skip peer and hostname verification
    if (isSSL)
    {
        curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYPEER, 0L);
        curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYHOST, 0L);
    }
}

int MyProxy::Writer(char *data,
    uint32_t size,
    uint32_t nmemb,
    std::string *writerData)
{
    if (writerData == nullptr)
    {
        Log("ERR: %s writerData is null!", __func__);
        return 0;
    }
    int len = size * nmemb;
    if ((writerData->size() + len) > m_MAXBUF)
    {
        Log("ERR: %s writerData size over MAXBUF!", __func__);
        return 0;
    }
    writerData->append(data, len);
    return len;
}

我想用libcurl实现一个代理,它可以获取给定url(https)的内容。此外,它需要是线程安全的。但是当我使用 pthreads 创建 200 个线程来测试我的代码时,有时会发生段错误。如何解决这个问题?与 sRetContent(std::string) 有关系吗?谢谢!

错误:双重释放或损坏 (!prev): 0x0ac72840 ***分段错误

我的理解是,如果您使用的是https(看起来您是),libcurl 不是线程安全的,因为它使用的是底层 ssl 库。有关更多信息,请参阅 libcurl 文档和 OpenSSL 文档。

例如,如果你的libcurl是用OpenSSL编译的,那么你必须初始化一些回调函数,否则你可能会遇到问题。这是你需要做的事情(在Windows上编译):

#include <curl/curl.h>
#include <openssl/crypto.h>
void win32_locking_callback(int mode, int type, const char *file, int line)
{
    if (mode & CRYPTO_LOCK)
    {
        WaitForSingleObject(lock_cs[type],INFINITE);
    }
    else
    {
        ReleaseMutex(lock_cs[type]);
    }
}
void thread_setup(void)
{
    int i;
    lock_cs=(HANDLE*)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(HANDLE));
    for (i=0; i<CRYPTO_num_locks(); i++)
    {
        lock_cs[i]=CreateMutex(NULL,FALSE,NULL);
    }
    CRYPTO_set_locking_callback((void (*)(int,int,const char *,int))win32_locking_callback);
}
void thread_cleanup(void)
{
    int i;
    CRYPTO_set_locking_callback(NULL);
    for (i=0; i<CRYPTO_num_locks(); i++)
        CloseHandle(lock_cs[i]);
    OPENSSL_free(lock_cs);
}

我总是在调用curl_global_init(CURL_GLOBAL_ALL)后呼叫thread_setup()然后在我调用 curl_global_cleanup() 之前thread_cleanup()。

我经常在负载测试场景中将这种代码与 libcurl 一起使用,并且从未遇到任何问题。如果你继续遇到问题,这不是libcurl,而是在你的代码中没有正确完成的事情。

只要

你遵守规则,libcurl 就是线程安全的