Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

大量のレコードを Export するとメモリを食いつぶし OS ごとフリーズしてしまう不具合の修正 #1145

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

大量のレコードを Export するとメモリを食いつぶし OS ごとフリーズしてしまう不具合の修正 #1145

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8ff9b04
#1132 大量のレコードをエクスポートするのではなく, バッチ処理でエクスポートするように修正
K-Haruto Oct 31, 2024
a28ab37
エクスポート対象のデータ数と、エクスポートされたデータ数の数が合わない不具合の修正
K-Haruto Oct 31, 2024
993f7cb
1回のクエリで取得するデータ数上限を設定
K-Haruto Nov 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 57 additions & 35 deletions modules/Vtiger/actions/ExportData.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ function process(Vtiger_Request $request) {

private $moduleInstance;
private $focus;
private $exportBatchLimit = 10_000; // 1回のクエリで取得するデータ数上限

/**
* Function exports the data based on the mode
Expand All @@ -44,47 +45,58 @@ function ExportData(Vtiger_Request $request) {
$this->moduleFieldInstances = $this->moduleFieldInstances($moduleName);
$this->focus = CRMEntity::getInstance($moduleName);

$query = $this->getExportQuery($request);
$result = $db->pquery($query, array());

$redirectedModules = array('Users', 'Calendar');
if($request->getModule() != $moduleName && in_array($moduleName, $redirectedModules) && !$this->moduleCall){
$handlerClass = Vtiger_Loader::getComponentClassName('Action', 'ExportData', $moduleName);
$handler = new $handlerClass();
$handler->ExportData($request);
return;
}
$translatedHeaders = $this->getHeaders();
$entries = array();
for ($j = 0; $j < $db->num_rows($result); $j++) {
$entries[] = $this->sanitizeValues($db->fetchByAssoc($result, $j));
}

//エクスポート時の選択肢項目の翻訳処理
for ($i = 0; $i < $j; $i++){
$moduleModel = Vtiger_Module_Model::getInstance($moduleName);
$moduleFields = $moduleModel->getFields();
foreach ($entries[$i] as $columnName => $fieldValue) {
$fieldModel = $moduleFields[$columnName];
if(empty($fieldModel)){
foreach($moduleFields as $key => $fieldinfo){
if($fieldinfo->column == $columnName){
$fieldName = $fieldinfo->name;
$batchoffset = 0;
while(true){ // 取得するデータが無い場合に終了
$request->set('batchoffset', $batchoffset);
$query = $this->getExportQuery($request);
$result = $db->pquery($query, array());
$entries = array();
for ($j = 0; $j < $db->num_rows($result); $j++) {
$entries[] = $this->sanitizeValues($db->fetchByAssoc($result, $j));
}
if(empty($entries)){
break;
}

//エクスポート時の選択肢項目の翻訳処理
for ($i = 0; $i < $j; $i++){
$moduleModel = Vtiger_Module_Model::getInstance($moduleName);
$moduleFields = $moduleModel->getFields();
foreach ($entries[$i] as $columnName => $fieldValue) {
$fieldModel = $moduleFields[$columnName];
if(empty($fieldModel)){
foreach($moduleFields as $key => $fieldinfo){
if($fieldinfo->column == $columnName){
$fieldName = $fieldinfo->name;
}
}
$fieldModel = $moduleFields[$fieldName];
}
$fieldModel = $moduleFields[$fieldName];
}

$fieldDataType = ($fieldModel) ? $fieldModel->getFieldDataType() : '';
$fieldDataType = ($fieldModel) ? $fieldModel->getFieldDataType() : '';

if ($fieldDataType == 'picklist') {
$entries[$i][$columnName] = vtranslate($fieldValue,$moduleName);
if ($fieldDataType == 'picklist') {
$entries[$i][$columnName] = vtranslate($fieldValue,$moduleName);
}
}
}

if($batchoffset == 0){
$translatedHeaders = $this->getHeaders();
$this->output($request, $translatedHeaders, $entries);
}else{
$this->output($request, '', $entries);
}
$batchoffset = $batchoffset + $this->exportBatchLimit; // オフセットの更新
}

$this->output($request, $translatedHeaders, $entries);
}

public function getHeaders() {
Expand Down Expand Up @@ -205,6 +217,10 @@ function getExportQuery(Vtiger_Request $request) {
case 'ExportAllData' : if ($orderBy && $orderByFieldModel) {
$query .= ' ORDER BY '.$queryGenerator->getOrderByColumn($orderBy).' '.$sortOrder;
}
$batchOffset = $request->get('batchoffset');
if(isset($batchOffset)){
$query .= ' LIMIT '.$this->exportBatchLimit.' OFFSET '.$batchOffset;
}
break;

case 'ExportCurrentPage' : $pagingModel = new Vtiger_Paging_Model();
Expand Down Expand Up @@ -239,6 +255,10 @@ function getExportQuery(Vtiger_Request $request) {
if ($orderBy && $orderByFieldModel) {
$query .= ' ORDER BY '.$queryGenerator->getOrderByColumn($orderBy).' '.$sortOrder;
}
$batchOffset = $request->get('batchoffset');
if(isset($batchOffset)){
$query .= ' LIMIT '.$this->exportBatchLimit.' OFFSET '.$batchOffset;
}
break;


Expand Down Expand Up @@ -272,16 +292,18 @@ function output($request, $headers, $entries) {
$fileName = str_replace(',', '_', $fileName);
$exportType = $this->getExportContentType($request);

header("Content-Disposition:attachment;filename=$fileName.csv");
header("Content-Type:$exportType;charset=UTF-8");
header("Expires: Mon, 31 Dec 2000 00:00:00 GMT" );
header("Last-Modified: " . gmdate("D, d M Y H:i:s") . " GMT" );
header("Cache-Control: post-check=0, pre-check=0", false );

$header = implode("\",\"", $headers);
$header = "\"" .$header;
$header .= "\"\r\n";
echo chr(0xEF).chr(0xBB).chr(0xBF).$header;
if(!empty($headers)){ // 最初のバッチ処理で実行
header("Content-Disposition:attachment;filename=$fileName.csv");
header("Content-Type:$exportType;charset=UTF-8");
header("Expires: Mon, 31 Dec 2000 00:00:00 GMT" );
header("Last-Modified: " . gmdate("D, d M Y H:i:s") . " GMT" );
header("Cache-Control: post-check=0, pre-check=0", false );

$header = implode("\",\"", $headers);
$header = "\"" .$header;
$header .= "\"\r\n";
echo chr(0xEF).chr(0xBB).chr(0xBF).$header;
}

$cnt = 0;
foreach($entries as $row) {
Expand Down