process.c

Go to the documentation of this file.

/***************************************************************
 Copyright (C) 2013 Hewlett-Packard Development Company, L.P.
 Copyright (C) 2014-2015,2019 Siemens AG

 This program is free software; you can redistribute it and/or
 modify it under the terms of the GNU General Public License
 version 2 as published by the Free Software Foundation.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License along
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

 ***************************************************************/
#include "maintagent.h"

/**********  Globals  *************/
PGconn* pgConn = NULL;        
fo_dbManager* dbManager;      

PGresult * PQexecCheck(int exitNumber, char *SQL, char *file, const int line)
{
  PGresult *result;
  result = PQexec(pgConn, SQL);
  if (fo_checkPQcommand(pgConn, result, SQL, file, line)) {
    PQclear(result);
    exitNow(exitNumber);
  }
  return result;
}

FUNCTION void PQexecCheckClear(int exitNumber, char *SQL, char *file, const int line)
{
  PGresult *result;
  result = PQexecCheck(exitNumber, SQL, file, line);
  PQclear(result);
}

FUNCTION void vacAnalyze()
{
  long startTime, endTime;
  char *SQL = "VACUUM ANALYZE";

  startTime = (long)time(0);

  /* Vacuum and Analyze */
  PQexecCheckClear(-110, SQL, __FILE__, __LINE__);

  endTime = (long)time(0);
  printf("Vacuum Analyze took %ld seconds\n", endTime-startTime);

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}

FUNCTION void validateFolders()
{
  PGresult* result; // the result of the database access
  char *countTuples;

  char *invalidUploadRefs = "DELETE FROM foldercontents WHERE foldercontents_mode = 2 AND child_id NOT IN (SELECT upload_pk FROM upload)";
  char *invalidUploadtreeRefs = "DELETE FROM foldercontents WHERE foldercontents_mode = 4 AND child_id NOT IN (SELECT uploadtree_pk FROM uploadtree)";
  char *unRefFolders = "DELETE FROM folder WHERE folder_pk \
                        NOT IN (SELECT child_id FROM foldercontents WHERE foldercontents_mode = 1) AND folder_pk != '1'";
  long startTime, endTime;

  startTime = (long)time(0);

  /* Remove folder contents with invalid upload references */
  result = PQexecCheck(-120, invalidUploadRefs, __FILE__, __LINE__);
  countTuples = PQcmdTuples(result);
  PQclear(result);
  printf("%s Invalid folder upload References\n", countTuples);

  /* Remove folder contents with invalid uploadtree references */
  result = PQexecCheck(-121, invalidUploadtreeRefs, __FILE__, __LINE__);
  countTuples = PQcmdTuples(result);
  PQclear(result);
  printf("%s Invalid folder uploadtree References\n", countTuples);

  /* Remove unreferenced folders */
  result = PQexecCheck(-122, unRefFolders, __FILE__, __LINE__);
  countTuples = PQcmdTuples(result);
  PQclear(result);
  printf("%s unreferenced folders\n", countTuples);

  endTime = (long)time(0);
  printf("Validate folders took %ld seconds\n", endTime-startTime);

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}

FUNCTION void verifyFilePerms(int fix)
{
/*
  long StartTime, EndTime;
  char *RepoPath;

  StartTime = (long)time(0);

  RepoPath = fo_sysconfig("FOSSOLOGY", "path");
  if (stat(RepoPath, &statbuf) == -1)
  {
  }

  EndTime = (long)time(0);
  printf("Verify File Permissions took %ld seconds\n", EndTime-StartTime);
*/
  LOG_NOTICE("Verify file permissions is not implemented yet");

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}

FUNCTION void removeUploads()
{
  PGresult* result; // the result of the database access
  char *countTuples;
  long startTime, endTime;

  char *SQL = "DELETE FROM upload WHERE upload_pk  \
               IN (SELECT upload_fk FROM uploadtree WHERE parent IS NULL AND pfile_fk IS NULL)  \
               OR (upload_pk NOT IN (SELECT upload_fk FROM uploadtree) \
                 AND (expire_action IS NULL OR expire_action != 'd') AND pfile_fk IS NOT NULL)";

  startTime = (long)time(0);

  result = PQexecCheck(-130, SQL, __FILE__, __LINE__);
  countTuples = PQcmdTuples(result);
  PQclear(result);

  endTime = (long)time(0);
  printf("%s Uploads with no pfiles (%ld seconds)\n", countTuples, endTime-startTime);

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}

FUNCTION void removeTemps()
{
  PGresult* result;
  int row;
  int countTuples;
  int droppedCount = 0;
  char SQLBuf[MAXSQL];
  long startTime, endTime;

  char *SQL = "SELECT table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE'"
              "AND table_schema = 'public' AND (table_name SIMILAR TO '^metaanalysis_[[:digit:]]+$'"
              "OR table_name SIMILAR TO '^delup_%');";

  startTime = (long)time(0);

  result = PQexec(pgConn, SQL);
  if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) exitNow(-140);
  countTuples = PQntuples(result);
  /* Loop through the temp table names, dropping the tables */
  for (row = 0; row < countTuples; row++) {
    snprintf(SQLBuf, MAXSQL, "DROP TABLE %s", PQgetvalue(result, row, 0));
    PQexecCheckClear(-141, SQLBuf, __FILE__, __LINE__);
    droppedCount++;
  }
  PQclear(result);

  endTime = (long)time(0);
  printf("%d Orphaned temp tables were dropped (%ld seconds)\n", droppedCount, endTime-startTime);

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}

FUNCTION void processExpired()
{
/*
  PGresult* result; // the result of the database access
  int numrows;             // generic return value
  long StartTime, EndTime;

  StartTime = (long)time(0);

  EndTime = (long)time(0);
  printf("Process expired uploads took %ld seconds\n", EndTime-StartTime);
*/
  LOG_NOTICE("Process expired uploads is not implemented yet");

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}

FUNCTION void removeOrphanedFiles()
{
/*
  PGresult* result; // the result of the database access
  int numrows;             // generic return value
  long StartTime, EndTime;

  StartTime = (long)time(0);

  EndTime = (long)time(0);
  printf("Remove orphaned files from the repository took %ld seconds\n", EndTime-StartTime);
*/
  LOG_NOTICE("Remove orphaned files from the repository is not implemented yet");

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}

FUNCTION void deleteOrphanGold()
{
/*
  PGresult* result; // the result of the database access
  int numrows;             // generic return value
  long StartTime, EndTime;

  StartTime = (long)time(0);

  EndTime = (long)time(0);
  printf("Remove orphaned files from the repository took %ld seconds\n", EndTime-StartTime);
*/
  LOG_NOTICE("Remove orphaned gold files from the repository is not implemented yet");

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}

FUNCTION void normalizeUploadPriorities()
{
  long startTime, endTime;

  char *SQL1 = "CREATE TEMPORARY TABLE tmp_upload_prio(ordprio serial, uploadid int, groupid int)";
  char *SQL2 = "INSERT INTO tmp_upload_prio (uploadid, groupid) (SELECT upload_fk uploadid, group_fk groupid FROM upload_clearing ORDER BY priority ASC);";
  char *SQL3 = "UPDATE upload_clearing SET priority = ordprio FROM tmp_upload_prio WHERE uploadid=upload_fk AND group_fk=groupid;";

  startTime = (long)time(0);

  PQexecCheckClear(-180, SQL1, __FILE__, __LINE__);
  PQexecCheckClear(-181, SQL2, __FILE__, __LINE__);
  PQexecCheckClear(-182, SQL3, __FILE__, __LINE__);

  endTime = (long)time(0);
  printf("Normalized upload priorities (%ld seconds)\n", endTime-startTime);

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}

FUNCTION void reIndexAllTables()
{
  PGresult* result; // the result of the database access
  char SQLBuf[MAXSQL];
  long startTime, endTime;
  char *SQL= "SELECT table_catalog FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'public' AND (table_name SIMILAR TO 'upload%') LIMIT 1";

  startTime = (long)time(0);

  result = PQexec(pgConn, SQL);
  if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) exitNow(-190);

  memset(SQLBuf,'\0',sizeof(SQLBuf));
  snprintf(SQLBuf,sizeof(SQLBuf),"REINDEX DATABASE %s;", PQgetvalue(result, 0, 0));
  PQclear(result);
  PQexecCheckClear(-191, SQLBuf, __FILE__, __LINE__);

  endTime = (long)time(0);
  printf("Time taken for reindexing the database : %ld seconds\n", endTime-startTime);

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}

FUNCTION void removeOrphanedRows()
{
  PGresult* result; // the result of the database access
  char *countTuples;
  long startTime, endTime;

  char *SQL1 = "DELETE FROM uploadtree UT "
               " WHERE NOT EXISTS ( "
               "  SELECT 1 "
               "  FROM upload U "
               "  WHERE UT.upload_fk = U.upload_pk "
               " );";

  char *SQL2 = "DELETE FROM clearing_decision AS CD "
               " WHERE NOT EXISTS ( "
               "  SELECT 1 "
               "  FROM uploadtree UT  "
               "  WHERE CD.uploadtree_fk = UT.uploadtree_pk "
               " ) AND CD.scope = '0';";

  char *SQL3 = "DELETE FROM clearing_event CE "
               " WHERE NOT EXISTS ( "
               "  SELECT 1 "
               "  FROM uploadtree UT  "
               "  WHERE CE.uploadtree_fk = UT.uploadtree_pk "
               " ) AND NOT EXISTS ( "
               "  SELECT 1 "
               "  FROM clearing_decision CD"
               "  WHERE CD.uploadtree_fk = CE.uploadtree_fk "
               "  AND CD.scope = '1'"
               " );";

  char *SQL4 = "DELETE FROM clearing_decision_event CDE"
               " WHERE NOT EXISTS ( "
               "  SELECT 1 "
               "  FROM clearing_event CE  "
               "  WHERE CE.clearing_event_pk = CDE.clearing_event_fk "
               " );";

  char *SQL5 = "DELETE FROM obligation_map OM "
               " WHERE NOT EXISTS ( "
               "  SELECT 1 "
               "  FROM license_ref LR  "
               "  WHERE OM.rf_fk = LR.rf_pk "
               " );";

  char *SQL6 = "DELETE FROM obligation_candidate_map OCM "
               " WHERE NOT EXISTS ( "
               "  SELECT 1 "
               "  FROM license_ref LR  "
               "  WHERE OCM.rf_fk = LR.rf_pk "
               " );";

  startTime = (long)time(0);

  result = PQexecCheck(-200, SQL1, __FILE__, __LINE__);
  countTuples = PQcmdTuples(result);
  PQclear(result);
  printf("%s Orphaned records have been removed from uploadtree table\n", countTuples);
  fo_scheduler_heart(1);

  result = PQexecCheck(-201, SQL2, __FILE__, __LINE__);
  countTuples = PQcmdTuples(result);
  PQclear(result);
  printf("%s Orphaned records have been removed from clearing_decision table\n", countTuples);
  fo_scheduler_heart(1);

  result = PQexecCheck(-202, SQL3, __FILE__, __LINE__);
  countTuples = PQcmdTuples(result);
  PQclear(result);
  printf("%s Orphaned records have been removed from clearing_event table\n", countTuples);
  fo_scheduler_heart(1);

  result = PQexecCheck(-203, SQL4, __FILE__, __LINE__);
  countTuples = PQcmdTuples(result);
  PQclear(result);
  printf("%s Orphaned records have been removed from clearing_decision_event table\n", countTuples);
  fo_scheduler_heart(1);

  result = PQexecCheck(-204, SQL5, __FILE__, __LINE__);
  countTuples = PQcmdTuples(result);
  PQclear(result);
  printf("%s Orphaned records have been removed from obligation_map table\n", countTuples);
  fo_scheduler_heart(1);

  result = PQexecCheck(-205, SQL6, __FILE__, __LINE__);
  countTuples = PQcmdTuples(result);
  PQclear(result);
  printf("%s Orphaned records have been removed from obligation_candidate_map table\n", countTuples);
  fo_scheduler_heart(1); // Tell the scheduler that we are alive and update item count

  endTime = (long)time(0);

  printf("Time taken for removing orphaned rows from database : %ld seconds\n", endTime-startTime);

  return;  // success
}


FUNCTION void removeOrphanedLogFiles()
{
  PGresult* result;
  PGresult* updateResult;
  int row;
  int countTuples;
  int removedCount = 0;
  int jobQueueId;
  char* logPath;
  long startTime, endTime;
  fo_dbManager_PreparedStatement* updateStatement;
  struct stat statbuf;

  char *SQL = "SELECT jq_pk, jq_log FROM job ja "
    "INNER JOIN job jb ON ja.job_upload_fk = jb.job_upload_fk "
    "INNER JOIN jobqueue jq ON jb.job_pk = jq.jq_job_fk "
    "WHERE ja.job_name = 'Delete' AND jq_log IS NOT NULL "
    "AND jq_log != 'removed';";

  startTime = (long)time(0);

  result = PQexec(pgConn, SQL);
  if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__))
  {
    exitNow(-140);
  }
  countTuples = PQntuples(result);

  updateStatement = fo_dbManager_PrepareStamement(dbManager,
    "updateRemovedLogFromJobqueue",
    "UPDATE jobqueue SET jq_log = 'removed' WHERE jq_pk = $1;", int);
  /* Loop through the logs found and delete them. Also update the database */
  for (row = 0; row < countTuples; row++)
  {
    fo_dbManager_begin(dbManager);
    jobQueueId = atoi(PQgetvalue(result, row, 0));
    logPath = PQgetvalue(result, row, 1);
    if (stat(logPath, &statbuf) == -1)
    {
      LOG_NOTICE("Log file '%s' does not exists", logPath);
    }
    else
    {
      remove(logPath);
      LOG_VERBOSE2("Removed file '%s'", logPath);
    }
    updateResult = fo_dbManager_ExecPrepared(updateStatement, jobQueueId);
    if (updateResult)
    {
      free(updateResult);
      removedCount++;
    }
    else
    {
      LOG_ERROR("Unable to update the value of jobqueue.jq_log to 'removed' "
        "for jq_pk = %d", jobQueueId);
    }
    fo_dbManager_commit(dbManager);
  }
  PQclear(result);

  endTime = (long)time(0);
  printf("%d / %d Orphaned log files were removed "
    "(%ld seconds)\n", removedCount, countTuples, endTime - startTime);

  fo_scheduler_heart(1);  // Tell the scheduler that we are alive and update item count
  return;  // success
}