stats/split-stats.php
author Dan
Sun, 07 Dec 2008 08:45:54 -0500
changeset 31 d75124700259
parent 15 5e2d1514ccd0
permissions -rw-r--r--
Timeout recovery should avoid getting the bot throttled now :)
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
     1
<?php
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
     2
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
     3
@set_time_limit(0);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
     4
@ini_set('memory_limit', '256M');
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
     5
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
     6
if ( $dir = @opendir('.') )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
     7
{
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
     8
  while ( $dh = @readdir($dir) )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
     9
  {
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    10
    if ( !preg_match('/^stats-data(-[0-9]+)\.php$/', $dh) )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    11
      continue;
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    12
    
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    13
    split_stats_file($dh);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    14
  }
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    15
  closedir($dir);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    16
}
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    17
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    18
function split_stats_file($file)
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    19
{
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    20
  echo "loading $file";
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    21
  
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    22
  require($file);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    23
  if ( !is_array($stats_data) )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    24
  {
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    25
    return false;
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    26
  }
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    27
  
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    28
  unlink($file);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    29
  
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    30
  echo "\rprocessing $file\n";
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    31
  
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    32
  $newdata = array();
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    33
  foreach ( $stats_data['messages'] as $channel => &$chandata )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    34
  {
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    35
    echo "  processing channel $channel\n";
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    36
    foreach ( $chandata as $i => $message )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    37
    {
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    38
      $message_day = gmdate('Ymd', $message['time']);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    39
      if ( !isset($newdata[$message_day]) )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    40
      {
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    41
        echo "\r    processing " . gmdate('Y-m-d', $message['time']);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    42
        $newdata[$message_day] = array(
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    43
          'messages' => array()
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    44
        );
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    45
        if ( isset($stats_data['counts']) )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    46
        {
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    47
          $newdata[$message_day]['counts'] = $stats_data['counts'];
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    48
        }
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    49
        if ( isset($stats_data['anonymous']) )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    50
        {
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    51
          $newdata[$message_day]['anonymous'] = $stats_data['anonymous'];
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    52
        }
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    53
      }
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    54
      if ( !isset($newdata[$message_day]['messages'][$channel]) )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    55
      {
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    56
        $newdata[$message_day]['messages'][$channel] = array();
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    57
      }
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    58
      $newdata[$message_day]['messages'][$channel][] = $message;
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    59
      unset($chandata[$i]);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    60
    }
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    61
    echo "\n";
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    62
  }
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    63
  foreach ( $newdata as $date => &$data )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    64
  {
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    65
    echo "\r  writing output for $date";
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    66
    write_stats_file("stats-data-$date.php", $data);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    67
  }
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    68
  echo "\n";
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    69
}
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    70
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    71
function write_stats_file($file, $data)
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    72
{
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    73
  $fp = @fopen($file, 'w');
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    74
  if ( !$fp )
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    75
    return false;
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    76
  
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    77
  ob_start();
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    78
  var_export($data);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    79
  $data = ob_get_contents();
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    80
  ob_end_clean();
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    81
  
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    82
  fwrite($fp, "<?php\n\$stats_data = $data;\n");
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    83
  fclose($fp);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    84
  unset($data);
5e2d1514ccd0 Made stats use multiple files to reduce memory usage
Dan
parents:
diff changeset
    85
}