OjoUtils.cc

Go to the documentation of this file.

/*
 * Copyright (C) 2019, Siemens AG
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
#include <iostream>

#include "OjoUtils.hpp"
#include "OjoAgent.hpp"

using namespace fo;

OjoState getState(DbManager &dbManager, OjoCliOptions &&cliOptions)
{
  int agentId = queryAgentId(dbManager);
  return OjoState(agentId, std::move(cliOptions));
}

OjoState getState(OjoCliOptions &&cliOptions)
{
  return OjoState(-1, std::move(cliOptions));
}

int queryAgentId(DbManager &dbManager)
{
  char* COMMIT_HASH = fo_sysconfig(AGENT_NAME, "COMMIT_HASH");
  char* VERSION = fo_sysconfig(AGENT_NAME, "VERSION");
  char *agentRevision;

  if (!asprintf(&agentRevision, "%s.%s", VERSION, COMMIT_HASH))
    bail(-1);

  int agentId = fo_GetAgentKey(dbManager.getConnection(), AGENT_NAME, 0,
      agentRevision, AGENT_DESC);
  free(agentRevision);

  if (agentId <= 0)
    bail(1);

  return agentId;
}

int writeARS(const OjoState &state, int arsId, int uploadId, int success,
    DbManager &dbManager)
{
  PGconn *connection = dbManager.getConnection();
  int agentId = state.getAgentId();

  return fo_WriteARS(connection, arsId, uploadId, agentId, AGENT_ARS, NULL,
    success);
}

void bail(int exitval)
{
  fo_scheduler_disconnect(exitval);
  exit(exitval);
}

bool processUploadId(const OjoState &state, int uploadId,
    OjosDatabaseHandler &databaseHandler, bool ignoreFilesWithMimeType)
{
  vector<unsigned long> fileIds = databaseHandler.queryFileIdsForUpload(
      uploadId, ignoreFilesWithMimeType);
  char const *repoArea = "files";

  bool errors = false;
#pragma omp parallel
  {
    OjosDatabaseHandler threadLocalDatabaseHandler(databaseHandler.spawn());

    size_t pFileCount = fileIds.size();
    OjoAgent agentObj = state.getOjoAgent();
#pragma omp for
    for (size_t it = 0; it < pFileCount; ++it)
    {
      if (errors)
        continue;

      unsigned long pFileId = fileIds[it];

      if (pFileId == 0)
        continue;

      char *fileName = threadLocalDatabaseHandler.getPFileNameForFileId(
        pFileId);
      char *filePath = NULL;
#pragma omp critical (repo_mk_path)
      filePath = fo_RepMkPath(repoArea, fileName);

      if (!filePath)
      {
        LOG_FATAL(
          AGENT_NAME" was unable to derive a file path for pfile %ld.  Check your HOSTS configuration.",
          pFileId);
        errors = true;
      }

      vector<ojomatch> identified;
      try
      {
        identified = agentObj.processFile(filePath, threadLocalDatabaseHandler);
      }
      catch (std::runtime_error &e)
      {
        LOG_FATAL("Unable to read %s.", e.what());
        continue;
      }

      if (!storeResultInDb(identified, threadLocalDatabaseHandler,
          state.getAgentId(), pFileId))
      {
        LOG_FATAL("Unable to store results in database for pfile %ld.",
          pFileId);
        bail(-20);
      }

      fo_scheduler_heart(1);
    }
  }

  return !errors;
}

bool storeResultInDb(const vector<ojomatch> &matches,
    OjosDatabaseHandler &databaseHandle, const int agent_fk, const int pfile_fk)
{
  if (!databaseHandle.begin())
  {
    return false;
  }

  size_t count = 0;
  if (matches.size() == 0)
  {
    OjoDatabaseEntry entry(-1, agent_fk, pfile_fk);
    databaseHandle.insertNoResultInDatabase(entry);
    return databaseHandle.commit();
  }
  for (auto m : matches)
  {
    OjoDatabaseEntry entry(m.license_fk, agent_fk, pfile_fk);

    if (entry.license_fk > 0)
    {
      ++count;
      unsigned long int fl_pk = databaseHandle.saveLicenseToDatabase(entry);
      if (!(fl_pk > 0) || !databaseHandle.saveHighlightToDatabase(m, fl_pk))
      {
        databaseHandle.rollback();
        return false;
      }
    }
    else
    {
      databaseHandle.insertNoResultInDatabase(entry);
    }
  }

  return databaseHandle.commit();
}

bool parseCliOptions(int argc, char **argv, OjoCliOptions &dest,
    std::vector<std::string> &fileNames, string &directoryToScan)
{
  boost::program_options::options_description desc(
    AGENT_NAME ": recognized options");
  desc.add_options()
    (
      "help,h", "shows help"
    )
    (
      "verbose,v", "increase verbosity"
    )
    (
      "files",
      boost::program_options::value<vector<string> >(),
      "files to scan"
    )
    (
      "json,J", "output JSON"
    )
    (
      "ignoreFilesWithMimeType,I", "ignoreFilesWithMimeType"
    )
    (
      "config,c",
      boost::program_options::value<string>(),
      "path to the sysconfigdir"
    )
    (
      "scheduler_start",
      "specifies, that the command was called by the scheduler"
    )
    (
      "userID",
      boost::program_options::value<int>(),
      "the id of the user that created the job (only in combination with --scheduler_start)"
    )
    (
      "groupID",
      boost::program_options::value<int>(),
      "the id of the group of the user that created the job (only in combination with --scheduler_start)"
    )
    (
      "jobId",
      boost::program_options::value<int>(),
      "the id of the job (only in combination with --scheduler_start)"
    )
    (
      "directory,d",
      boost::program_options::value<string>(),
      "directory to scan (recursive)"
    )
    ;

  boost::program_options::positional_options_description p;
  p.add("files", -1);

  boost::program_options::variables_map vm;

  try
  {
    boost::program_options::store(
      boost::program_options::command_line_parser(argc, argv).options(desc).positional(
        p).run(), vm);

    if (vm.count("help") > 0)
    {
      cout << desc << endl;
      exit(0);
    }

    if (vm.count("files"))
    {
      fileNames = vm["files"].as<std::vector<string> >();
    }

    unsigned long verbosity = vm.count("verbose");
    bool json = vm.count("json") > 0 ? true : false;
    bool  ignoreFilesWithMimeType = vm.count("ignoreFilesWithMimeType") > 0 ?  true : false;

    dest = OjoCliOptions(verbosity, json, ignoreFilesWithMimeType);

    if (vm.count("directory"))
    {
      if (vm.count("files"))
      {
        cout << "cannot pass files and directory at the same time" << endl;
        cout << desc << endl;
        fileNames.clear();
        return false;
      }
      directoryToScan = vm["directory"].as<std::string>();
    }

    return true;
  }
  catch (boost::bad_any_cast&)
  {
    cout << "wrong parameter type" << endl;
    cout << desc << endl;
    return false;
  }
  catch (boost::program_options::error&)
  {
    cout << "wrong command line arguments" << endl;
    cout << desc << endl;
    return false;
  }
}

void appendToJson(const std::string fileName,
    const std::pair<string, vector<ojomatch>> resultPair,
    bool &printComma)
{
  Json::Value result;
#if JSONCPP_VERSION_HEXA < ((1 << 24) | (4 << 16))
  // Use FastWriter for versions below 1.4.0
  Json::FastWriter jsonWriter;
#else
  // Since version 1.4.0, FastWriter is deprecated and replaced with
  // StreamWriterBuilder
  Json::StreamWriterBuilder jsonWriter;
  jsonWriter["commentStyle"] = "None";
  jsonWriter["indentation"] = "";
#endif
  if (resultPair.first.empty())
  {
    result["file"] = fileName;
    result["results"] = "Unable to read file";
  }
  else
  {
    vector<ojomatch> resultList = resultPair.second;
    Json::Value results;
    for (auto m : resultList)
    {
      Json::Value j;
      j["start"] = Json::Value::UInt(m.start);
      j["end"] = Json::Value::UInt(m.end);
      j["len"] = Json::Value::UInt(m.len);
      j["license"] = m.content;
      results.append(j);
    }
    result["file"] = fileName;
    result["results"] = results;
  }
  // Thread-Safety: output all matches JSON at once to STDOUT
#pragma omp critical (jsonPrinter)
  {
    if (printComma)
    {
      cout << "," << endl;
    }
    else
    {
      printComma = true;
    }
    string jsonString;
#if JSONCPP_VERSION_HEXA < ((1 << 24) | (4 << 16))
    // For version below 1.4.0, every writer append `\n` at end.
    // Find and replace it.
    jsonString = jsonWriter.write(result);
    jsonString.replace(jsonString.find("\n"), string("\n").length(), "");
#else
    // For version >= 1.4.0, \n is not appended.
    jsonString = Json::writeString(jsonWriter, result);
#endif
    cout << "  " << jsonString << flush;
  }
}

void printResultToStdout(const std::string fileName,
    const std::pair<string, vector<ojomatch>> resultPair)
{
  if (resultPair.first.empty())
  {
    cout << fileName << " :: Unable to read file" << endl;
    return;
  }
  stringstream ss;
  ss << fileName << " ::" << endl;
  // Output matches
  vector<ojomatch> resultList = resultPair.second;
  for (auto m : resultList)
  {
    ss << "\t[" << m.start << ':' << m.end << "]: '" << m.content << "'" << endl;
  }
  // Thread-Safety: output all matches (collected in ss) at once to cout
  cout << ss.str();
}