source: trunk/bin/alternc_stats_web.php @ 1396

#!/usr/bin/php4 -q 
<?php

/**
 * Statistiques webs par compte AlternC
 *
 * Ce script lit tous les fichiers de log apache et insère les
 * statistiques de visite par jour et par domaine / sous-domaine dans
 * la table stat_http
 *
 * On assume que:
 *
 * - on est appelé à traiter un fichier de log donné qu'une seule fois
 * - les fichiers de logs sont traités en ordre chronologique
 *
 * La façon correcte de faire ceci est de traiter access.log.0 juste
 * après la rotation.
 */

/* Configuration */

/**
 * Combien de lignes de logs traite-t-on par bloc
 *
 * 10000 conseillé, plus possible (à tester), moins vivement
 * déconseillé
 */
$each=10000;

/**
 * the maximum line length
 *
 * lines larger than this will not be parsed properly
 *
 * Was originally 2048, but was skipping too much of virus log
 * entries. We still have some with 4096, but much less (at worst
 * 1/10000 hits)
 */
@define('BUFSIZE', 4096);

/* end of configuration */

/**
 * never stop because of time
 */
@set_time_limit(0);

/**
 * bypass login checks
 */
include("/var/alternc/bureau/class/config_nochk.php");

// Libère le bureau !
alternc_shutdown();

$argv0 = array_shift($argv);

# crude getopt for portability
$verbose = $full = 0;
foreach ($argv as $pos => $arg) {
  switch ($arg) {
  case "-v":
    unset($argv[$pos]);
    $verbose = 1;
    break;
  case "-f":
    unset($argv[$pos]);
    $full = 1;
    @mysql_query("DELETE FROM stat_http");
    break;
  }
}

$total_hits = $total_good = 0;

// process all logfiles
foreach ($argv as $file) {

  if ($verbose) {
    echo "################################################\n";
    echo date("d/m/Y H:i:s")." : Traitement de $file\n";
    echo "################################################\n";
  }

  if (substr($file,-3)==".gz") {
    $open="gzopen";
    $gets="gzgets";
    $close="gzclose";
  } else {
    $open="fopen";
    $gets="fgets";
    $close="fclose";
  }

  if (($f = $open($file, "rb")) === FALSE) {
    die("impossible d'ouvrir le log $file");
  }

  $i = $l= $good = 0;

  $domstat=array(); // On stocke sous forme de clé "dom/day" => "hit or bandwith"
  $domuid=array();  // Cache des uids associés aux domaines.


  $months=array("Jan"=>"01","Feb"=>"02","Mar"=>"03","Apr"=>"04","May"=>"05","Jun"=>"06","Jul"=>"07","Aug"=>"08","Sep"=>"09","Oct"=>"10","Nov"=>"11","Dec"=>"12");

  /**
   * match "escaped quotes" or "everything except quotes"
   *
   * this might represent a significant performance hit
   */
  $noquote = '(?:(?:(?<=\\\)")|(?:[^"]))*';

  // Exemple de ligne apache : 
  // crawl18.dir.com - login [14/Jun/2004:06:38:47 +0200] "GET /modules/newbb?days=100 HTTP/1.0" 200 20156 "-" "Pompos/1.3 http://dir.com/pompos.html" 2 esperance-jeunes.org

  // this pattern should match it
  $pattern = '/^[^ ]* [^ ]* [^ ]* \\[([0-9]*)\\/([a-zA-Z]*)\\/([0-9]*):[0-9]*:[0-9]*:[0-9]* [^ ]* "'.$noquote.'" ([0-9-]*) ([0-9-]*) "'.$noquote.'" "'.$noquote.'" [0-9]* ([^ ]*)$/';

  // parse a line at a time
  while ($s=$gets($f,BUFSIZE)) {
    $s=trim($s);
    if (preg_match($pattern,$s,$mat)) {
      // ok, 1: jour  2: mois (english)  3: année  4: http result (200/404 ...) 5: taille  6: domaine
      // Ligne ok.
      //    echo "Ligne ok : ";
      //    for($j=1;$j<count($mat);$j++) echo "$j:".$mat[$j]." ";
      // On calcule le jour : (= yearmonthday)
      $day=$mat[3].$months[$mat[2]].$mat[1];
      $dom = mysql_escape_string(strtolower($mat[6]));
      $domstat[$dom."/".$day]["hit"]++;
      $domstat[$dom."/".$day]["size"]+=intval($mat[5]);
      $good++;
    } else {
      if ($verbose)
        printf("Can't parse: %.60s...\n", $s);
    }
    if ($i/100==intval($i/100)) { echo "."; flush(); }
    $i++; $l++;

    // insérer une fois que notre bloc de lignes est plein
    if ($i==$each) {
      if ($verbose) echo " $l lines read\nInserting... ";
      insert_lines($domstat, &$domuid);
      $domstat = array();
      $i=0;
    }
  }

  // insérer les lignes qui n'avaient pas rempli un bloc
  if ($verbose) echo "\nInserting remaining lines... ";
  insert_lines($domstat, &$domuid); 
  $domstat = array();

  $close($f);
 
  $total_hits += $l;
  $total_good += $good;

  if ($verbose) echo "\n$l lines read ($good good)\n";

}

// handy function to avoid a copy-paste
function insert_lines($domstat, &$domuid) {
  global $verbose;
  // on insère le tableau dans mysql.
  $update=0; $insert=0;
  foreach($domstat as $date => $stat) {
    preg_match("/^([^\\/]*)\\/(.*)$/",$date,$mat);
    $dom =$mat[1];
    $date = $mat[2];
    // on cherche l'uid de ce domaine
    if (!$domuid[$dom]) {
      list($domuid[$dom])=@mysql_fetch_array(mysql_query("SELECT compte FROM sub_domaines WHERE (domaine='".$dom."' AND sub='') OR (concat(sub,'.',domaine)='".$dom."');"));
    }
    // a-t-on déjà ce domaine ce jour ? 
    list($ct)=@mysql_fetch_array(mysql_query("SELECT COUNT(*) FROM stat_http WHERE day='".$date."' AND domain='".$dom."';"));
    if ($ct) {
      $sql="UPDATE stat_http SET hit= hit + '".$stat["hit"]."', size= size + '".$stat["size"]."' WHERE domain='".$dom."' AND day='".$date."';";
      $update++;
    } else {
      $sql="INSERT INTO stat_http SET uid='".$domuid[$dom]."',day='".$date."',domain='".$dom."', hit='".$stat["hit"]."', size='".$stat["size"]."';";
      $insert++;
    }
    mysql_query($sql) || die("Query failed: " . mysql_error());
  }
  if ($verbose) echo "$update updates and $insert inserts (uidcache size = ".count($domuid).") \n";

}

if ($verbose) echo "\n$total_hits lines read total ($total_good good)\n";

// check validity
if ($full) {
  list($res) = mysql_fetch_array(mysql_query("SELECT SUM(hit) FROM stat_http"));
  if ($res == $total_good) {
    echo "correct count: $res\n";
  } else {
    echo "difference between database hits and lines read: $res => $total_good\n";
  }
}

?>