As you can see below, there is a controller structure that pulls various data from Instagram and calculates the engagement rate after pulling this data. Although this structure works fast in small-scale accounts, it becomes very slow and inefficient when it comes to large accounts. I tried to speed it up by trying various things, such as using the yield method, but since I'm new to php, I'm not even sure if I should use yield in this code. Could you please help me on what to do? Thanks in advance.
this is my controller code
<?php
namespace App\Http\Controllers;
use Illuminate\Support\Facades\Http;
class splenperAPIController extends Controller
{
public function splenperAPI()
{
$maxId = '';
$response = Http::withHeaders([
'cookie' => 'sessionid=1796659686:4Ojj1py72bZKql:7; csrftoken=fPYrPRD1vHB7LdS0DjKzOK4kGo4uYK9f; ds_user_id=1796659686; ig_did=25114CCD-7A9D-4971-88F1-1E04796D9F14; ig_nrcb=1; mid=YiqZZQALAAEhkff6N5T2oovGOBkz; rur=01f73b7ee19feca3df296fc45ee75179c49dd54efb93233307b58f19bf707fea5d458fe7; shbid=01f7998eadf528233f0e7331b327e5a107134084c64e229af2cc6577b8a2ce862e3798da; shbts=01f7caf607a6877ca8407e9cb85cd175736a87f8f75e9076d5bddb32cbcbac6d5da45e89',
'x-ig-app-id' => '936619743392459',
'Content-Type' => 'application/json',
])->get('https://www.instagram.com/elmaligroup/?__a=1');
$response = $response->json();
$userId = $response['graphql']['user']['id'];
$followersCount = $response['graphql']['user']['edge_followed_by']['count'];
$count = 12;
$index = 0;
$isMoreAvailable = true;
$totalLikeCount = 0;
$totalCommentCount = 0;
while ($index < $count && $isMoreAvailable) {
$variables = json_encode([
'id' => $userId,
"after" => $maxId,
"first" => $count,
]);
$variables = urlencode($variables);
$response = Http::withHeaders([
'cookie' => 'sessionid=1796659686:4Ojj1py72bZKql:7; csrftoken=fPYrPRD1vHB7LdS0DjKzOK4kGo4uYK9f; ds_user_id=1796659686; ig_did=25114CCD-7A9D-4971-88F1-1E04796D9F14; ig_nrcb=1; mid=YiqZZQALAAEhkff6N5T2oovGOBkz; rur=01f73b7ee19feca3df296fc45ee75179c49dd54efb93233307b58f19bf707fea5d458fe7; shbid=01f7998eadf528233f0e7331b327e5a107134084c64e229af2cc6577b8a2ce862e3798da; shbts=01f7caf607a6877ca8407e9cb85cd175736a87f8f75e9076d5bddb32cbcbac6d5da45e89',
'x-ig-app-id' => '936619743392459',
'Content-Type' => 'application/json',
])->get('https://www.instagram.com/graphql/query/?query_hash=e769aa130647d2354c40ea6a439bfc08&variables=' . $variables);
for ($i = 0; $i < $count; $i ) {
if ($i == count($response->json()['data']['user']['edge_owner_to_timeline_media']['edges'])) {
break;
}
$totalCommentCount = $response->json()['data']['user']['edge_owner_to_timeline_media']['edges'][$i]['node']['edge_media_to_comment']['count'];
$totalLikeCount = $response->json()['data']['user']['edge_owner_to_timeline_media']['edges'][$i]['node']['edge_media_preview_like']['count'];
$userName = $response->json()['data']['user']['edge_owner_to_timeline_media']['edges'][$i]['node']['owner']['username'];
$index ;
}
$maxId = $response->json()['data']['user']['edge_owner_to_timeline_media']['page_info']['end_cursor'];
$isMoreAvailable = $response->json()['data']['user']['edge_owner_to_timeline_media']['page_info']['has_next_page'];
if ($isMoreAvailable) {
$index = 0;
}
}
$mediaCount = $response->json()['data']['user']['edge_owner_to_timeline_media']['count'];
echo $followersCount . '<br>';
echo $totalLikeCount . '<br>';
echo $totalCommentCount . '<br>';
echo ($totalLikeCount $totalCommentCount) . '<br>';
echo $userName . '<br>';
echo $userId . '<br>';
echo $mediaCount . '<br>';
$percent = ($totalLikeCount $totalCommentCount) / $followersCount * 100;
echo "engagementRate: " . number_format($percent, 2, ',', '.') . '%';
}
}
CodePudding user response：
I believe the big problem here is the number of HTTP requests you are doing. I think you must increase the number per request. It will decrease HTTP requests and probably speedup your code.
You set it on $count = 12;
I refactor your code and put $maxPerPage = 50 and tried to decrease some unnecessary counting and call for method $response->json().
$headers = [
'cookie' => 'sessionid=1796659686:4Ojj1py72bZKql:7; csrftoken=fPYrPRD1vHB7LdS0DjKzOK4kGo4uYK9f; ds_user_id=1796659686; ig_did=25114CCD-7A9D-4971-88F1-1E04796D9F14; ig_nrcb=1; mid=YiqZZQALAAEhkff6N5T2oovGOBkz; rur=01f73b7ee19feca3df296fc45ee75179c49dd54efb93233307b58f19bf707fea5d458fe7; shbid=01f7998eadf528233f0e7331b327e5a107134084c64e229af2cc6577b8a2ce862e3798da; shbts=01f7caf607a6877ca8407e9cb85cd175736a87f8f75e9076d5bddb32cbcbac6d5da45e89',
'x-ig-app-id' => '936619743392459',
'Content-Type' => 'application/json',
];
$response = Http::withHeaders($headers)->get('https://www.instagram.com/elmaligroup/?__a=1');
$response = $response->json();
$userId = $response['graphql']['user']['id'];
$followersCount = $response['graphql']['user']['edge_followed_by']['count'];
$maxPerPage = 50;
$endCursor = '';
$hasNextPage = true;
$totalLikeCount = 0;
$totalCommentCount = 0;
$userName = '';
$mediaCount = 0;
while ($hasNextPage) {
$variables = urlencode(json_encode([
"id" => $userId,
"after" => $endCursor,
"first" => $maxPerPage,
]));
$response = Http::withHeaders($headers)->get('https://www.instagram.com/graphql/query/?query_hash=e769aa130647d2354c40ea6a439bfc08&variables=' . $variables);
$edges = $response->json()['data']['user']['edge_owner_to_timeline_media'];
$totalOfEdges = count($edges['edges']);
for ($i = 0; $i < $totalOfEdges; $i ) {
$totalCommentCount = $edges['edges'][$i]['node']['edge_media_to_comment']['count'];
$totalLikeCount = $edges['edges'][$i]['node']['edge_media_preview_like']['count'];
$userName = $edges['edges'][$i]['node']['owner']['username'];
}
$endCursor = $edges['page_info']['end_cursor'];
$hasNextPage = $edges['page_info']['has_next_page'];
$mediaCount = $edges['count'];
}
echo $followersCount . '<br>';
echo $totalLikeCount . '<br>';
echo $totalCommentCount . '<br>';
echo ($totalLikeCount $totalCommentCount) . '<br>';
echo $userName . '<br>';
echo $userId . '<br>';
echo $mediaCount . '<br>';
$percent = ($totalLikeCount $totalCommentCount) / $followersCount * 100;
echo "engagementRate: " . number_format($percent, 2, ',', '.') . '%';
CodePudding user response：
i've made a basic benchmark with microtime and memory_get_usage so after (or while if you can tail laravel log) script runs you will be able to see some stats about timings and memory and overal stats after script finish, then update question and we can think about next steps
public function splenperAPI()
{
$workStart = microtime(true);
$headers = [
'cookie' => 'sessionid=1796659686:4Ojj1py72bZKql:7; csrftoken=fPYrPRD1vHB7LdS0DjKzOK4kGo4uYK9f; ds_user_id=1796659686; ig_did=25114CCD-7A9D-4971-88F1-1E04796D9F14; ig_nrcb=1; mid=YiqZZQALAAEhkff6N5T2oovGOBkz; rur=01f73b7ee19feca3df296fc45ee75179c49dd54efb93233307b58f19bf707fea5d458fe7; shbid=01f7998eadf528233f0e7331b327e5a107134084c64e229af2cc6577b8a2ce862e3798da; shbts=01f7caf607a6877ca8407e9cb85cd175736a87f8f75e9076d5bddb32cbcbac6d5da45e89',
'x-ig-app-id' => '936619743392459',
'Content-Type' => 'application/json',
];
$maxId = '';
$response = Http::withHeaders($headers)->get('https://www.instagram.com/elmaligroup/?__a=1');
$response = $response->json();
$userId = $response['graphql']['user']['id'];
$followersCount = $response['graphql']['user']['edge_followed_by']['count'];
$count = 12;
$index = 0;
$isMoreAvailable = true;
$totalLikeCount = 0;
$totalCommentCount = 0;
// debug vars
$requestsCount = 0;
$slowestRequestTime = 0;
$totalRequestsTime = 0;
$totalProcessingDataTime = 0;
// debug vars end
while ($index < $count && $isMoreAvailable) {
$variables = urlencode(
json_encode([
'id' => $userId,
"after" => $maxId,
"first" => $count,
])
);
// we'll log requests durations
$startTime = microtime(true);
$response = Http::withHeaders($headers)
->get("https://www.instagram.com/graphql/query/?query_hash=e769aa130647d2354c40ea6a439bfc08&variables=$variables");
$endTime = microtime(true);
$requestsCount ;
$requestTime = round($endTime - $startTime, 6);
$totalRequestsTime = $requestTime;
if ($requestTime > $slowestRequestTime) $slowestRequestTime = $requestTime;
// also we'll log processing requested data duration
$startTime = microtime(true);
for ($i = 0; $i < $count; $i ) {
if ($i == count($response->json()['data']['user']['edge_owner_to_timeline_media']['edges'])) {
break;
}
$totalCommentCount = $response->json()['data']['user']['edge_owner_to_timeline_media']['edges'][$i]['node']['edge_media_to_comment']['count'];
$totalLikeCount = $response->json()['data']['user']['edge_owner_to_timeline_media']['edges'][$i]['node']['edge_media_preview_like']['count'];
$userName = $response->json()['data']['user']['edge_owner_to_timeline_media']['edges'][$i]['node']['owner']['username'];
$index ;
}
$maxId = $response->json()['data']['user']['edge_owner_to_timeline_media']['page_info']['end_cursor'];
$isMoreAvailable = $response->json()['data']['user']['edge_owner_to_timeline_media']['page_info']['has_next_page'];
if ($isMoreAvailable) {
$index = 0;
}
$endTime = microtime(true);
$processingDataTime = round($endTime - $startTime, 6);
$totalProcessingDataTime = $processingDataTime;
Log::debug('speed', [
'request time' => $requestTime,
'process data time' => $processingDataTime,
'current memory usage (MB)' => memory_get_usage() / 1048576
]);
}
$workDuration = round(microtime(true) - $workStart, 6);
// and the final log to get total data
Log::debug('speed', [
'total requests' => $requestsCount,
'total network time' => $totalRequestsTime,
'slowest' => $slowestRequestTime,
'average' => round($totalRequestsTime / $requestsCount, 6),
'total processing time' => $totalProcessingDataTime,
'total work time' => $workDuration,
'memory peak (MB)' => memory_get_peak_usage() / 1048576
]);
$mediaCount = $response->json()['data']['user']['edge_owner_to_timeline_media']['count'];
echo $followersCount . '<br>';
echo $totalLikeCount . '<br>';
echo $totalCommentCount . '<br>';
echo ($totalLikeCount $totalCommentCount) . '<br>';
echo $userName . '<br>';
echo $userId . '<br>';
echo $mediaCount . '<br>';
$percent = ($totalLikeCount $totalCommentCount) / $followersCount * 100;
echo "engagementRate: " . number_format($percent, 2, ',', '.') . '%';
}