I am working with an IOT system and trying to improve our data ingestion. Currently we are iterating over an array of URLs with different ports and sending one curl request at a time. I would like to use multicurl to get around this bottleneck but I am getting only getting errors as a response. Here’s my code (IP addresses redacted)
$url = 'http://XXX.XXX.XX.XXX/getvar.csv';
$ports = [8101,8102,8103,8104];
foreach ($ports as $port) {
$worker = curl_init($curl_url);
curl_setopt_array($worker, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_POST => true,
CURLOPT_HEADER => 0,
CURLOPT_TIMEOUT => 8,
CURLOPT_POSTFIELDS => $content,
CURLOPT_PORT => $port
]);
curl_multi_add_handle($mh, $worker);
}
$i=0;
for (; ;) {
$still_running = null;
echo "<p>exec multi curl $i</p>";
do {
echo "<p>curl multi exec</p>";
$err = curl_multi_exec($mh, $still_running);
} while ($err === CURLM_CALL_MULTI_PERFORM);
if ($err !== CURLM_OK) {
$description = curl_strerror(curl_errno($mh));
echo "<p>Error description: $description</p>";
// handle curl multi error?
}
if ($still_running < 1) {
echo "<p>curl multi finished</p>";
// all downloads completed
break;
}
// some haven't finished downloading, sleep until more data arrives:
curl_multi_select($mh, 8);
$i++;
}
$curl_time_stop = microtime(true);
$curl_duration += $curl_time_stop - $curl_time_start;
echo "CURL finished";
$results = [];
while (false !== ($info = curl_multi_info_read($mh))) {
if ($info["result"] !== CURLE_OK) {
echo "Error ".curl_strerror($info["result"]);
// handle download error?
}
$results[curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL)] =
curl_multi_getcontent($info["handle"]);
curl_multi_remove_handle($mh, $info["handle"]);
curl_close($info["handle"]);
}
curl_multi_close($mh);
var_dump($results);
Here’s the errors/response I am getting
Error Server returned nothing (no headers, no data) results int(1) int(52) resource(29) of type (curl)
From what I can tell error code 52 corresponds to CURLE_GOT_NOTHING.
Is there something I am missing? Or is there a difference in the protocols used by multi-curl vs curl?
[Edit: Add verbose output]
Here’s the result when I use CURLOPT_VERBOSE=>1
* Trying 1XX.XX.XX.XX:8101... * Found bundle for host 1XX.XX.XX.XX: 0x2625a00 [serially] * Server doesn't support multiplex (yet) * Hostname 1XX.XX.XX.XX was found in DNS cache * Trying 1XX.XX.XX.XX:8101... *** Connected to 1XX.XX.XX.XX (1XX.XX.XX.XX) port 8101 (#0) > POST /getvar.csv? HTTP/1.1 Host: 1XX.XX.XX.XX:8101 Accept: */* Content-Length: 248 Content-Type: application/x-www-form-urlencoded** * Mark bundle as not supporting multiuse < HTTP/1.1 200 OK < Content-Type: text/csv; charset="utf-8" < Server: embedded HTTPD < Expires: 1 JAN 2013 00:00:00 GMT < Last-Modified: 4 MAR 2021 12:10:09 GMT < Cache-Control: no-cache < Transfer-Encoding: chunked * Connection #0 to host 1XX.XX.XX.XX left intact * Empty reply from server * Closing connection 25
Answer
I found an external issue to my code, but am posting the final, working code here for future reference.
Multi-curl does not like iterating over the port number so I had to bake the port into the curl URL and iterate over an array of URLs.
The $content is not required for the multicurl to work, but it is to get data from the URLs I’m calling.
<?php
$lists=[
"LocalBoardTemp"
];
$url = 'http://XXX.XXX.XXX.XXX:';
$ports = [8101,8102,8103,8104,8105,8106,8107,8108,8109];
foreach ($ports as $port) {
$curl_urls[] = $url.$port.'/getvar.csv?';
}
$content = '';
foreach ($lists as $list1) {
$content = $content . "&name=" . $list1;
}
$mh = curl_multi_init();
$fp = fopen(dirname(__FILE__).'/curl_errorlog.txt', 'w');
$i=0;
foreach ($curl_urls as $url) {
$worker = curl_init($url);
curl_setopt_array($worker, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_POST => true,
CURLOPT_HEADER => 0,
CURLOPT_TIMEOUT => 8,
CURLOPT_POSTFIELDS => $content,
CURLOPT_VERBOSE=>1,
CURLOPT_STDERR => $fp
// ,
// CURLOPT_PORT => $port
]);
curl_multi_add_handle($mh, $worker);
}
for (; ;) {
$still_running = null;
echo "<p>exec multi curl $i</p>";
do {
echo "<p>curl multi exec</p>";
$err = curl_multi_exec($mh, $still_running);
} while ($err === CURLM_CALL_MULTI_PERFORM);
if ($err !== CURLM_OK) {
$description = curl_strerror(curl_errno($mh));
echo "<p>Error description: $description</p>";
// handle curl multi error?
}
if ($still_running < 1) {
echo "<p>curl multi finished</p>";
// all downloads completed
break;
}
// some haven't finished downloading, sleep until more data arrives:
curl_multi_select($mh, 8);
$i++;
}
echo "CURL finished";
$results = [];
$j = 0;
while (false !== ($info = curl_multi_info_read($mh))) {
if ($info["result"] !== CURLE_OK) {
echo "Error ".curl_strerror($info["result"]);
// handle download error?
}
echo "<p>results ";
foreach ($info as $x) {
var_dump($x);
}
echo "</p>";
$results[curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL)] = curl_multi_getcontent($info["handle"]);
echo "<p>result index $j". curl_multi_getcontent($info["handle"])."</p>";
curl_multi_remove_handle($mh, $info["handle"]);
curl_close($info["handle"]);
$j++;
}
curl_multi_close($mh);
echo "<p>All results dump: ";
var_dump($results);
echo "</p>";
fclose($fp);
echo "finished";
This has been cobbled together from several tutorials and stack overflow answers, so I’m sure there is a more efficient way of doing this, but it works.