00001 <?php
00009 function convertLinks() {
00010 global $wgDBtype;
00011 if( $wgDBtype == 'postgres' ) {
00012 wfOut( "Links table already ok on Postgres.\n" );
00013 return;
00014 }
00015
00016 wfOut( "Converting links table to ID-ID...\n" );
00017
00018 global $wgLang, $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname;
00019 global $noKeys, $logPerformance, $fh;
00020
00021 $tuplesAdded = $numBadLinks = $curRowsRead = 0; #counters etc
00022 $totalTuplesInserted = 0; # total tuples INSERTed into links_temp
00023
00024 $reportCurReadProgress = true; #whether or not to give progress reports while reading IDs from cur table
00025 $curReadReportInterval = 1000; #number of rows between progress reports
00026
00027 $reportLinksConvProgress = true; #whether or not to give progress reports during conversion
00028 $linksConvInsertInterval = 1000; #number of rows per INSERT
00029
00030 $initialRowOffset = 0;
00031 #$finalRowOffset = 0; # not used yet; highest row number from links table to process
00032
00033 # Overwrite the old links table with the new one. If this is set to false,
00034 # the new table will be left at links_temp.
00035 $overwriteLinksTable = true;
00036
00037 # Don't create keys, and so allow duplicates in the new links table.
00038 # This gives a huge speed improvement for very large links tables which are MyISAM. (What about InnoDB?)
00039 $noKeys = false;
00040
00041
00042 $logPerformance = false; # output performance data to a file
00043 $perfLogFilename = "convLinksPerf.txt";
00044 #--------------------------------------------------------------------
00045
00046 $dbw = wfGetDB( DB_MASTER );
00047 list ($cur, $links, $links_temp, $links_backup) = $dbw->tableNamesN( 'cur', 'links', 'links_temp', 'links_backup' );
00048
00049 $res = $dbw->query( "SELECT l_from FROM $links LIMIT 1" );
00050 if ( $dbw->fieldType( $res, 0 ) == "int" ) {
00051 wfOut( "Schema already converted\n" );
00052 return;
00053 }
00054
00055 $res = $dbw->query( "SELECT COUNT(*) AS count FROM $links" );
00056 $row = $dbw->fetchObject($res);
00057 $numRows = $row->count;
00058 $dbw->freeResult( $res );
00059
00060 if ( $numRows == 0 ) {
00061 wfOut( "Updating schema (no rows to convert)...\n" );
00062 createTempTable();
00063 } else {
00064 if ( $logPerformance ) { $fh = fopen ( $perfLogFilename, "w" ); }
00065 $baseTime = $startTime = getMicroTime();
00066 # Create a title -> cur_id map
00067 wfOut( "Loading IDs from $cur table...\n" );
00068 performanceLog ( "Reading $numRows rows from cur table...\n" );
00069 performanceLog ( "rows read vs seconds elapsed:\n" );
00070
00071 $dbw->bufferResults( false );
00072 $res = $dbw->query( "SELECT cur_namespace,cur_title,cur_id FROM $cur" );
00073 $ids = array();
00074
00075 while ( $row = $dbw->fetchObject( $res ) ) {
00076 $title = $row->cur_title;
00077 if ( $row->cur_namespace ) {
00078 $title = $wgLang->getNsText( $row->cur_namespace ) . ":$title";
00079 }
00080 $ids[$title] = $row->cur_id;
00081 $curRowsRead++;
00082 if ($reportCurReadProgress) {
00083 if (($curRowsRead % $curReadReportInterval) == 0) {
00084 performanceLog( $curRowsRead . " " . (getMicroTime() - $baseTime) . "\n" );
00085 wfOut( "\t$curRowsRead rows of $cur table read.\n" );
00086 }
00087 }
00088 }
00089 $dbw->freeResult( $res );
00090 $dbw->bufferResults( true );
00091 wfOut( "Finished loading IDs.\n\n" );
00092 performanceLog( "Took " . (getMicroTime() - $baseTime) . " seconds to load IDs.\n\n" );
00093 #--------------------------------------------------------------------
00094
00095 # Now, step through the links table (in chunks of $linksConvInsertInterval rows),
00096 # convert, and write to the new table.
00097 createTempTable();
00098 performanceLog( "Resetting timer.\n\n" );
00099 $baseTime = getMicroTime();
00100 wfOut( "Processing $numRows rows from $links table...\n" );
00101 performanceLog( "Processing $numRows rows from $links table...\n" );
00102 performanceLog( "rows inserted vs seconds elapsed:\n" );
00103
00104 for ($rowOffset = $initialRowOffset; $rowOffset < $numRows; $rowOffset += $linksConvInsertInterval) {
00105 $sqlRead = "SELECT * FROM $links ";
00106 $sqlRead = $dbw->limitResult($sqlRead, $linksConvInsertInterval,$rowOffset);
00107 $res = $dbw->query($sqlRead);
00108 if ( $noKeys ) {
00109 $sqlWrite = array("INSERT INTO $links_temp (l_from,l_to) VALUES ");
00110 } else {
00111 $sqlWrite = array("INSERT IGNORE INTO $links_temp (l_from,l_to) VALUES ");
00112 }
00113
00114 $tuplesAdded = 0; # no tuples added to INSERT yet
00115 while ( $row = $dbw->fetchObject($res) ) {
00116 $fromTitle = $row->l_from;
00117 if ( array_key_exists( $fromTitle, $ids ) ) { # valid title
00118 $from = $ids[$fromTitle];
00119 $to = $row->l_to;
00120 if ( $tuplesAdded != 0 ) {
00121 $sqlWrite[] = ",";
00122 }
00123 $sqlWrite[] = "($from,$to)";
00124 $tuplesAdded++;
00125 } else { # invalid title
00126 $numBadLinks++;
00127 }
00128 }
00129 $dbw->freeResult($res);
00130 #wfOut( "rowOffset: $rowOffset\ttuplesAdded: $tuplesAdded\tnumBadLinks: $numBadLinks\n" );
00131 if ( $tuplesAdded != 0 ) {
00132 if ($reportLinksConvProgress) {
00133 wfOut( "Inserting $tuplesAdded tuples into $links_temp..." );
00134 }
00135 $dbw->query( implode("",$sqlWrite) );
00136 $totalTuplesInserted += $tuplesAdded;
00137 if ($reportLinksConvProgress)
00138 wfOut( " done. Total $totalTuplesInserted tuples inserted.\n" );
00139 performanceLog( $totalTuplesInserted . " " . (getMicroTime() - $baseTime) . "\n" );
00140 }
00141 }
00142 wfOut( "$totalTuplesInserted valid titles and $numBadLinks invalid titles were processed.\n\n" );
00143 performanceLog( "$totalTuplesInserted valid titles and $numBadLinks invalid titles were processed.\n" );
00144 performanceLog( "Total execution time: " . (getMicroTime() - $startTime) . " seconds.\n" );
00145 if ( $logPerformance ) { fclose ( $fh ); }
00146 }
00147 #--------------------------------------------------------------------
00148
00149 if ( $overwriteLinksTable ) {
00150 $dbConn = Database::newFromParams( $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname );
00151 if (!($dbConn->isOpen())) {
00152 wfOut( "Opening connection to database failed.\n" );
00153 return;
00154 }
00155 # Check for existing links_backup, and delete it if it exists.
00156 wfOut( "Dropping backup links table if it exists..." );
00157 $dbConn->query( "DROP TABLE IF EXISTS $links_backup", DB_MASTER);
00158 wfOut( " done.\n" );
00159
00160 # Swap in the new table, and move old links table to links_backup
00161 wfOut( "Swapping tables '$links' to '$links_backup'; '$links_temp' to '$links'..." );
00162 $dbConn->query( "RENAME TABLE links TO $links_backup, $links_temp TO $links", DB_MASTER );
00163 wfOut( " done.\n\n" );
00164
00165 $dbConn->close();
00166 wfOut( "Conversion complete. The old table remains at $links_backup;\n" );
00167 wfOut( "delete at your leisure.\n" );
00168 } else {
00169 wfOut( "Conversion complete. The converted table is at $links_temp;\n" );
00170 wfOut( "the original links table is unchanged.\n" );
00171 }
00172 }
00173
00174 #--------------------------------------------------------------------
00175
00176 function createTempTable() {
00177 global $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname;
00178 global $noKeys;
00179 $dbConn = Database::newFromParams( $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname );
00180
00181 if (!($dbConn->isOpen())) {
00182 wfOut( "Opening connection to database failed.\n" );
00183 return;
00184 }
00185 $links_temp = $dbConn->tableName( 'links_temp' );
00186
00187 wfOut( "Dropping temporary links table if it exists..." );
00188 $dbConn->query( "DROP TABLE IF EXISTS $links_temp");
00189 wfOut( " done.\n" );
00190
00191 wfOut( "Creating temporary links table..." );
00192 if ( $noKeys ) {
00193 $dbConn->query( "CREATE TABLE $links_temp ( " .
00194 "l_from int(8) unsigned NOT NULL default '0', " .
00195 "l_to int(8) unsigned NOT NULL default '0')");
00196 } else {
00197 $dbConn->query( "CREATE TABLE $links_temp ( " .
00198 "l_from int(8) unsigned NOT NULL default '0', " .
00199 "l_to int(8) unsigned NOT NULL default '0', " .
00200 "UNIQUE KEY l_from(l_from,l_to), " .
00201 "KEY (l_to))");
00202 }
00203 wfOut( " done.\n\n" );
00204 }
00205
00206 function performanceLog( $text ) {
00207 global $logPerformance, $fh;
00208 if ( $logPerformance ) {
00209 fwrite( $fh, $text );
00210 }
00211 }
00212
00213 function getMicroTime() { # return time in seconds, with microsecond accuracy
00214 list($usec, $sec) = explode(" ", microtime());
00215 return ((float)$usec + (float)$sec);
00216 }