推进：内存耗尽错误

Question

对于较大的批处理操作，我遇到内存限制错误，有时由于运行大型脚本而没有批处理操作。我尝试增加 php memory_limit 但由于某种原因它没有反映在错误消息中，并且内存错误是相同的。

这不是 Neo4j 错误或 Php Wrapper 错误！！

这是一个 Propel(ORM) 错误。看来 propel 和 Php 有内存泄漏。即使在取消设置对象、禁用实例池并清除对象的所有引用之后，错误仍然存​​在。

例如：

Propel::disableInstancePooling() ;
$newObj->clearAllReferences(true);
unset($newObj);

致命错误：第 101 行 /Applications/MAMP/htdocs/queremos/vendor/everyman/neo4jphp/lib/Everyman/Neo4j/Transport/Curl.php 中允许的内存大小为 1073741824 字节已用尽（试图分配 298112365 字节）

示例代码：使用 php symfony 1.4

 <?php
require("vendor/autoload.php");
?>

<?php
use Aws\S3\S3Client;

class populateGraphDbTask extends sfBaseTask
{
  protected function configure()
  {
    $this->namespace        = '';
    $this->name             = 'populateGraphDb';
    $this->briefDescription = 'populateGraphDb - populate graph db to test';
    $this->detailedDescription = <<<EOF
The [populateGraphDb|INFO] task does things.
Call it with:

  [php symfony populateGraphDb|INFO]
EOF;
  }

  protected function execute($arguments = array(), $options = array())
  {
Propel::disableInstancePooling() ;
    // initialize the database connection
    $databaseManager = new sfDatabaseManager($this->configuration);    
    $connection = $databaseManager->getDatabase($options['connection'])->getConnection();

    // PREVENT THE ERROR - The "default" context does not exist.
    $config = ProjectConfiguration::getApplicationConfiguration(
      $options['application'],
      $options['env'],
      $options['connection']
    );
    sfContext::createInstance($config);

    // Connecting to the default port 7474 on localhost
    $neo = new Everyman\Neo4j\Client();
    print_r($neo->getServerInfo());



    //---create node indexes

    //create the Facebook Page nodes
    $fbPageIdx = new Everyman\Neo4j\Index\NodeIndex($neo, 'fbPageIdx');
    $fbPageIdx->save();

//create unique constrainst on FbPage node
    $queryString = "CREATE CONSTRAINT ON (page:FbPage) ASSERT page.like_id IS UNIQUE";
    $query = new Everyman\Neo4j\Cypher\Query($neo, $queryString);
    $result = $query->getResultSet();
    //var_dump($result);

    echo "-> creating FbPages nodes in a batch transaction \n";
    $page = 1;
    $page_size = 10000;
    $start_page = 75;

    $last_id = 0;

    $connection = Propel::getConnection();
    $sql = "SELECT  Table_Name, table_rows as count
    FROM    INFORMATION_SCHEMA.TABLES
    WHERE   TABLE_TYPE = 'BASE TABLE'
    AND     TABLE_SCHEMA = 'sample.db'
    AND   Table_name = 'cliente_fblike'";
    $statement = $connection->prepare($sql);
    $statement->execute();
    $result_count = $statement->fetchAll();
    $count = $result_count[0]["count"];

    //$fblikes = ClienteFblikeQuery::create()->paginate($page, $page_size);
    $num_pages = ceil($count / $page_size);

    //var_dump('-> count - '.$count);
    //var_dump('-> num_pages - '.$num_pages);

    echo "\n";
    echo "-> count="."$count \n";
    for($i=$start_page; $i<$num_pages; $i++)
    {
      //start batch
      echo "starting batch \n";
      $batch = $neo->startBatch();
      echo "-> on_Page - ".$i." of - ".$num_pages."\n";
      echo "-> last_id - ".$last_id."\n";

      $fblikes = ClientFblikeQuery::create()->paginate($i, $page_size);

      foreach($fblikes as $fblike)
      {
        $last_id = $fblike->getId();
        echo '.';

        //create the FbPage node, if duplicate it will fail with error
        //var_dump('-> creating node('.$fblike->getId().'): FbPage - '.$fblike->getLikeId().'='.$fblike->getName().'='.$fblike->getCategory());
        //echo "-> creating node(".$fblike->getId().'): FbPage - '.$fblike->getLikeId().'='.$fblike->getName().'='.$fblike->getCategory()."\n";
        //echo '( )';

        $nw = $neo->makeNode();
        $nw->setProperty('type', 'FbPage');
        $nw->setProperty('category', $fblike->getCategory());
        $nw->setProperty('pagename', $fblike->getName());
        $nw->setProperty('like_id', $fblike->getLikeId());
        $nw->setProperty('created_at', $fblike->getCreatedAt());
        //$nw->setProperty('updated_at', $fblike->getUpdatedAt());
        $saved_node = $nw->save();

        //add to the index
        $fbPageIdx->add($nw, 'like_id', $fblike->getLikeId());
        $fbPageIdx->add($nw, 'category', $fblike->getCategory());
        $fbPageIdx->add($nw, 'pagename', $fblike->getName());
        //echo '(!)';
        //var_dump('! node('.$fblike->getId().') exists: FbPage - '.$fblike->getLikeId().'='.$fblike->getName().'='.$fblike->getCategory());
        //var_dump($match);
      }

      echo "\n commiting batch \n";
      $batch->commit();
    }
  }

Answer 1

这个问题可以通过以下方式解决：

启用 php 垃圾回收。 禁用 Propel 实例池。 清除所有对象引用。 未设置的对象。 调用垃圾收集器。

while(loop_db_objs)
{
  gc_enable();
  Propel::disableInstancePooling();

  //create/update your objects

  $newObj->clearAllReferences(true);
  unset($newObj);
  gc_collect_cycles();
}