74 include_once(__DIR__ .
"/spdx2utils.php");
76 include_once(__DIR__ .
"/version.php");
77 include_once(__DIR__ .
"/services.php");
145 function __construct()
148 $args = getopt(
"", array(self::OUTPUT_FORMAT_KEY.
'::'));
154 parent::__construct(
$agentName, AGENT_VERSION, AGENT_REV);
156 $this->uploadDao = $this->container->get(
'dao.upload');
157 $this->clearingDao = $this->container->get(
'dao.clearing');
158 $this->licenseDao = $this->container->get(
'dao.license');
159 $this->
dbManager = $this->container->get(
'db.manager');
160 $this->renderer = $this->container->get(
'twig.environment');
161 $this->renderer->setCache(
false);
163 $this->agentSpecifLongOptions[] = self::UPLOAD_ADDS.
':';
164 $this->agentSpecifLongOptions[] = self::OUTPUT_FORMAT_KEY.
':';
175 return $dbManager->booleanFromDb($lic->getSpdxCompatible());
186 if ((!array_key_exists(self::OUTPUT_FORMAT_KEY,
$args)
187 ||
$args[self::OUTPUT_FORMAT_KEY] ===
"")
188 && array_key_exists(self::UPLOAD_ADDS,
$args)) {
191 if (!array_key_exists(self::UPLOAD_ADDS,
$args) ||
$args[self::UPLOAD_ADDS] ===
"") {
206 if (array_key_exists(self::OUTPUT_FORMAT_KEY,
$args)) {
207 $possibleOutputFormat =
trim(
$args[self::OUTPUT_FORMAT_KEY]);
208 if (in_array($possibleOutputFormat, explode(
',',self::AVAILABLE_OUTPUT_FORMATS))) {
209 $this->outputFormat = $possibleOutputFormat;
212 $this->licenseMap =
new LicenseMap($this->
dbManager, $this->groupId, LicenseMap::REPORT,
true);
216 $additionalUploadIds = array_key_exists(self::UPLOAD_ADDS,
$args) ? explode(
',',
$args[self::UPLOAD_ADDS]) : array();
217 $packageIds = array($uploadId);
218 foreach ($additionalUploadIds as $additionalId) {
220 $packageIds[] = $additionalId;
223 $this->
writeReport($packageNodes, $packageIds, $uploadId);
234 $prefix = $this->outputFormat .
"-";
236 switch ($this->outputFormat) {
238 $postfix =
".xml" . $postfix;
243 $prefix = $prefix .
"copyright-";
246 return $prefix . $partname . $postfix;
258 if ($this->filebasename == null) {
259 $fileName = strtoupper($this->outputFormat).
"_".$packageName.
'_'.time();
260 switch ($this->outputFormat) {
262 $fileName = $fileName .
"-spdx.rdf";
265 $fileName = $fileName .
".spdx";
268 $fileName = $fileName .
".txt";
271 $this->filebasename = $fileName;
284 $fileBase = $SysConf[
'FOSSOLOGY'][
'path'].
"/report/";
296 $url=$SysConf[
'SYSCONFIG'][
'FOSSologyURL'];
297 if (substr( $url, 0, 4 ) !==
"http") {
311 $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
312 $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName);
324 $this->addCopyrightResults($filesWithLicenses, $uploadId);
327 $upload = $this->uploadDao->getUpload($uploadId);
328 $fileNodes = $this->
generateFileNodes($filesWithLicenses, $upload->getTreeTableName(), $uploadId);
330 $mainLicenseIds = $this->clearingDao->getMainLicenseIds($uploadId, $this->groupId);
331 $mainLicenses = array();
332 foreach ($mainLicenseIds as $licId) {
333 $reportedLicenseId = $this->licenseMap->getProjectedId($licId);
334 $this->includedLicenseIds[$reportedLicenseId] =
true;
335 $mainLicenses[] = $this->licenseMap->getProjectedShortname($reportedLicenseId);
338 if (strcmp($this->outputFormat,
"dep5")!==0) {
342 $hashes = $this->uploadDao->getUploadHashes($uploadId);
344 'packageId' => $uploadId,
346 'packageName' => $upload->getFilename(),
347 'uploadName' => $upload->getFilename(),
348 'sha1' => $hashes[
'sha1'],
349 'md5' => $hashes[
'md5'],
350 'sha256' => $hashes[
'sha256'],
352 'mainLicenses' => $mainLicenses,
354 'licenseComments' => $licenseComment,
355 'fileNodes' => $fileNodes)
366 $clearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBounds, $this->groupId);
368 $filesWithLicenses = array();
369 $clearingsProceeded = 0;
370 foreach ($clearingDecisions as $clearingDecision) {
371 $clearingsProceeded += 1;
372 if (($clearingsProceeded&2047)==0) {
375 if ($clearingDecision->getType() == DecisionTypes::IRRELEVANT) {
379 foreach ($clearingDecision->getClearingEvents() as $clearingEvent) {
380 $clearingLicense = $clearingEvent->getClearingLicense();
381 if ($clearingLicense->isRemoved()) {
386 $filesWithLicenses[$clearingDecision->getUploadTreeId()][
'comment'][] = $clearingLicense->getComment();
387 if ($clearingEvent->getReportinfo()) {
388 $customLicenseText = $clearingEvent->getReportinfo();
389 $reportedLicenseShortname = $this->licenseMap->getProjectedShortname($this->licenseMap->getProjectedId($clearingLicense->getLicenseId())) .
390 '-' . md5($customLicenseText);
391 $this->includedLicenseIds[$reportedLicenseShortname] = $customLicenseText;
392 $filesWithLicenses[$clearingDecision->getUploadTreeId()][
'concluded'][] = $reportedLicenseShortname;
394 $reportedLicenseId = $this->licenseMap->getProjectedId($clearingLicense->getLicenseId());
395 $this->includedLicenseIds[$reportedLicenseId] =
true;
396 $filesWithLicenses[$clearingDecision->getUploadTreeId()][
'concluded'][] = $this->licenseMap->getProjectedShortname($reportedLicenseId);
400 return $filesWithLicenses;
415 if (!array_key_exists($key, $filesWithLicenses)) {
416 $filesWithLicenses[$key][
'files']=array();
417 $filesWithLicenses[$key][
'copyrights']=array();
419 if (empty($copyrights)) {
420 $copyrights = array();
422 $filesWithLicenses[$key][
'files'][$file] = $fullPath;
423 foreach ($copyrights as $copyright) {
424 if (!in_array($copyright, $filesWithLicenses[$key][
'copyrights'])) {
425 $filesWithLicenses[$key][
'copyrights'][] = $copyright;
438 $licensesWithFiles = array();
439 $treeDao = $this->container->get(
'dao.tree');
441 foreach ($filesWithLicenses as $fileId=>$licenses) {
442 $filesProceeded += 1;
443 if (($filesProceeded&2047)==0) {
446 $fullPath = $treeDao->getFullPath($fileId,$treeTableName,0);
447 if (!empty($licenses[
'concluded']) && count($licenses[
'concluded'])>0) {
448 $this->
toLicensesWithFilesAdder($licensesWithFiles,$licenses[
'concluded'],$licenses[
'copyrights'],$fileId,$fullPath);
450 if (!empty($licenses[
'scanner']) && count($licenses[
'scanner']) > 0) {
452 if ($licenses[
'isCleared']) {
453 $msgLicense =
"None (scanners found: " . $implodedLicenses .
")";
455 $msgLicense =
"NoLicenseConcluded (scanners found: " . $implodedLicenses .
")";
458 if ($licenses[
'isCleared']) {
459 $msgLicense =
"None";
461 $msgLicense =
"NoLicenseConcluded";
467 return $licensesWithFiles;
479 $scannerAgents = array_keys($this->agentNames);
480 $scanJobProxy =
new ScanJobProxy($this->container->get(
'dao.agent'), $uploadId);
481 $scanJobProxy->createAgentStatus($scannerAgents);
482 $scannerIds = $scanJobProxy->getLatestSuccessfulAgentIds();
483 if (empty($scannerIds)) {
487 $stmt = __METHOD__ .
'.scanner_findings';
488 $sql =
"SELECT DISTINCT uploadtree_pk,rf_fk FROM $tableName ut, license_file 489 WHERE ut.pfile_fk=license_file.pfile_fk AND rf_fk IS NOT NULL AND agent_fk=any($1)";
490 $param = array(
'{'.implode(
',',$scannerIds).
'}');
491 if ($tableName ==
'uploadtree_a') {
492 $param[] = $uploadId;
493 $sql .=
" AND upload_fk=$".count($param);
496 $sql .=
" GROUP BY uploadtree_pk,rf_fk";
498 $res = $this->
dbManager->execute($stmt,$param);
499 while ($row=$this->
dbManager->fetchArray($res)) {
500 $reportedLicenseId = $this->licenseMap->getProjectedId($row[
'rf_fk']);
501 $shortName = $this->licenseMap->getProjectedShortname($reportedLicenseId);
502 if ($shortName !=
'Void') {
503 if ($shortName !=
'No_license_found') {
504 $filesWithLicenses[$row[
'uploadtree_pk']][
'scanner'][] = $shortName;
506 $filesWithLicenses[$row[
'uploadtree_pk']][
'scanner'][] =
"";
508 $this->includedLicenseIds[$reportedLicenseId] =
true;
514 $func =
function($scannerId) use (
$agentDao)
516 return $agentDao->getAgentName($scannerId).
" (".
$agentDao->getAgentRev($scannerId).
")";
518 $scannerNames = array_map($func, $scannerIds);
519 return "licenseInfoInFile determined by Scanners:\n - ".implode(
"\n - ",$scannerNames);
527 protected function addCopyrightResults(&$filesWithLicenses, $uploadId)
531 $copyrightDao = $this->container->get(
'dao.copyright');
533 $scanJobProxy =
new ScanJobProxy($this->container->get(
'dao.agent'),
536 $scanJobProxy->createAgentStatus(array(
$agentName));
537 $selectedScanners = $scanJobProxy->getLatestSuccessfulAgentIds();
538 if (!array_key_exists(
$agentName, $selectedScanners)) {
541 $latestAgentId = $selectedScanners[
$agentName];
542 $extrawhere =
' agent_fk='.$latestAgentId;
544 $uploadtreeTable = $this->uploadDao->getUploadtreeTableName($uploadId);
545 $allScannerEntries = $copyrightDao->getScannerEntries(
'copyright', $uploadtreeTable, $uploadId, $type=
'statement', $extrawhere);
546 $allEditedEntries = $copyrightDao->getEditedEntries(
'copyright_decision', $uploadtreeTable, $uploadId, $decisionType=null);
547 foreach ($allScannerEntries as $finding) {
548 $filesWithLicenses[$finding[
'uploadtree_pk']][
'copyrights'][] =
\convertToUTF8($finding[
'content'],
false);
550 foreach ($allEditedEntries as $finding) {
551 $filesWithLicenses[$finding[
'uploadtree_pk']][
'copyrights'][] =
\convertToUTF8($finding[
'textfinding'],
false);
563 array(UploadTreeProxy::OPT_SKIP_THESE => UploadTreeProxy::OPT_SKIP_ALREADY_CLEARED,
564 UploadTreeProxy::OPT_ITEM_FILTER =>
"AND (lft BETWEEN ".$itemTreeBounds->
getLeft().
" AND ".$itemTreeBounds->
getRight().
")",
567 'already_cleared_uploadtree' . $itemTreeBounds->
getUploadId());
570 $filesThatShouldStillBeCleared = $alreadyClearedUploadTreeView->getNonArtifactDescendants($itemTreeBounds);
571 $alreadyClearedUploadTreeView->unmaterialize();
573 $uploadTreeIds = array_keys($filesWithLicenses);
574 foreach ($uploadTreeIds as $uploadTreeId) {
575 $filesWithLicenses[$uploadTreeId][
'isCleared'] =
false == array_key_exists($uploadTreeId,$filesThatShouldStillBeCleared);
585 $upload = $this->uploadDao->getUpload($uploadId);
586 $packageName = $upload->getFilename();
588 $this->uri = $this->
getUri($packageName);
589 $this->filename = $this->
getFileName($packageName);
598 protected function writeReport(&$packageNodes, $packageIds, $uploadId)
600 $fileBase = dirname($this->filename);
602 if (!is_dir($fileBase)) {
603 mkdir($fileBase, 0777,
true);
608 if (strcmp($this->outputFormat,
"dep5")!==0) {
613 'documentName' => $fileBase,
615 'userName' => $this->container->get(
'dao.user')->getUserName($this->userId) .
" (" . $this->container->get(
'dao.user')->getUserEmail($this->userId) .
")",
616 'organisation' =>
'',
617 'packageNodes' => $packageNodes,
618 'packageIds' => $packageIds,
619 'licenseTexts' => $licenseTexts)
624 $message = preg_replace(
'/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/',
'?',$message);
626 file_put_contents($this->filename, $message);
638 $this->
dbManager->insertTableRow(
'reportgen',
639 array(
'upload_fk'=>$uploadId,
'job_fk'=>
$jobId,
'filepath'=>$fileName),
651 return $this->renderer->loadTemplate($templateName)->render($vars);
663 if (strcmp($this->outputFormat,
"dep5") !== 0) {
680 $treeDao = $this->container->get(
'dao.tree');
685 foreach ($filesWithLicenses as $fileId=>$licenses) {
686 $filesProceeded += 1;
687 if (($filesProceeded & 2047) == 0) {
688 $this->
heartbeat($filesProceeded - $lastValue);
689 $lastValue = $filesProceeded;
691 $hashes = $treeDao->getItemHashes($fileId);
692 $fileName = $treeDao->getFullPath($fileId, $treeTableName, 0);
693 if (!is_array($licenses[
'concluded'])) {
694 $licenses[
'concluded'] = array();
696 if (!is_array($licenses[
'scanner'])) {
697 $licenses[
'scanner'] = array();
701 if (!$stateWoInfos ||
702 ($stateWoInfos && (!empty($licenses[
'concluded']) || (!empty($licenses[
'scanner']) && !empty($licenses[
'scanner'][0])) || !empty($licenses[
'copyrights'])))) {
703 $dataTemplate = array(
705 'sha1' => $hashes[
'sha1'],
706 'md5' => $hashes[
'md5'],
707 'sha256' => $hashes[
'sha256'],
709 'fileName' => $fileName,
710 'fileDirName' => dirname($fileName),
711 'fileBaseName' => basename($fileName),
712 'isCleared' => $licenses[
'isCleared'],
716 'copyrights' => $licenses[
'copyrights'],
717 'licenseCommentState' => $stateComment
725 $this->
heartbeat($filesProceeded - $lastValue);
743 foreach ($licensesWithFiles as $licenseId=>$entry) {
744 $filesProceeded += count($entry[
'files']);
745 if ($filesProceeded&(~2047) > $lastStep) {
746 $this->
heartbeat($filesProceeded - $lastValue);
747 $lastStep = $filesProceeded&(~2047) + 2048;
748 $lastValue = $filesProceeded;
752 if (strrpos($licenseId,
"NoLicenseConcluded (scanners found: ", -strlen($licenseId)) !==
false) {
753 $comment = substr($licenseId,20,strlen($licenseId)-21);
754 $licenseId =
"NoLicenseConcluded";
755 } elseif (strrpos($licenseId,
"None (scanners found: ", -strlen($licenseId)) !==
false) {
756 $comment = substr($licenseId,6,strlen($licenseId)-7);
761 'fileNames'=>$entry[
'files'],
762 'license'=>$licenseId,
763 'copyrights'=>$entry[
'copyrights'],
764 'comment'=>$comment));
766 $this->
heartbeat($filesProceeded - $lastValue);
776 $licenseTexts = array();
777 $licenseViewProxy =
new LicenseViewProxy($this->groupId,array(LicenseViewProxy::OPT_COLUMNS=>array(
'rf_pk',
'rf_shortname',
'rf_fullname',
'rf_text')));
778 $this->
dbManager->prepare($stmt=__METHOD__, $licenseViewProxy->getDbViewQuery());
781 while ($row=$this->
dbManager->fetchArray($res)) {
782 if (array_key_exists($row[
'rf_pk'], $this->includedLicenseIds)) {
783 $licenseTexts[$row[
'rf_shortname']] = array(
784 'text' => $row[
'rf_text'],
785 'name' => $row[
'rf_fullname'] ?: $row[
'rf_shortname']);
788 foreach ($this->includedLicenseIds as $license => $customText) {
789 if (
true !== $customText) {
790 $licenseTexts[$license] = array(
791 'text' => $customText,
796 return $licenseTexts;
813 $param[] = $upload->
getId();
816 $stmt .=
'.'.$upload->getTreeTableName();
819 $sql =
"SELECT STRING_AGG(lower_sha1,'') concat_sha1 FROM 820 (SELECT LOWER(pfile_sha1) lower_sha1 FROM pfile, $sql pfile_fk=pfile_pk ORDER BY pfile_sha1) templist";
821 $filelistPack = $this->
dbManager->getSingleRow($sql,$param,$stmt);
823 return sha1($filelistPack[
'concat_sha1']);
834 $sql =
"SELECT ri_spdx_selection FROM report_info WHERE upload_fk = $1";
835 $getCommentState = $this->
dbManager->getSingleRow($sql, array($uploadId), __METHOD__.
'.SPDX_license_comment');
836 if (!empty($getCommentState[
'ri_spdx_selection'])) {
837 $getCommentStateSingle = explode(
',', $getCommentState[
'ri_spdx_selection']);
838 if ($getCommentStateSingle[$key] ===
"checked") {
846 $agent->scheduler_connect();
847 $agent->run_scheduler_event_loop();
848 $agent->scheduler_disconnect(0);
generateFileNodesByFiles($filesWithLicenses, $treeTableName, $uploadId)
For each file, generate the nodes by files.
heartbeat($newProcessed)
Send hear beat to the scheduler.
static preWorkOnArgsFlp($args, $key1, $key2)
For a given set of arguments assign $args[$key1] and $args[$key2].
addScannerResults(&$filesWithLicenses, ItemTreeBounds $itemTreeBounds)
Attach finding agents to the files and return names of scanners.
getVerificationCode(Upload $upload)
Get a unique identifier for a given upload.
generateFileNodes($filesWithLicenses, $treeTableName, $uploadId)
Generate report nodes for files.
Structure of an Agent with all required parameters.
int jobId
The id of the job.
getLicenseTexts()
Get the license texts from fossology.
getSPDXReportConf($uploadId, $key)
Get spdx license comment state for a given upload.
Namespace used by SPDX2 agent.
updateReportTable($uploadId, $jobId, $fileName)
Update the reportgen table with new report path.
processUploadId($uploadId)
Given an upload ID, process the items in it.
getTemplateFile($partname)
Get TWIG template file based on output format.
Wrapper class for license map.
const DEFAULT_OUTPUT_FORMAT
Default output format.
getFileBasename($packageName)
Generate report basename based on upload name.
getUri($packageName)
Get the URI for the given package.
getFileName($packageName)
Get absolute path for report.
renderString($templateName, $vars)
Render a twig template.
const UPLOAD_ADDS
Argument for additional uploads.
static addPrefixOnDemandKeys($licenses, $spdxValidityChecker=null)
Add prefix to license keys.
getFilesWithLicensesFromClearings(ItemTreeBounds $itemTreeBounds)
Given an ItemTreeBounds, get the files with clearings.
renderPackage($uploadId)
Given an upload id, render the report string.
static addPrefixOnDemandList($licenses, $spdxValidityChecker=null)
Add prefix to license list.
generateFileNodesByLicenses($filesWithLicenses, $treeTableName)
For each file, generate the nodes by licenses.
fo_dbManager * dbManager
fo_dbManager object
preWorkOnArgs($args)
Parse arguments.
toLicensesWithFiles(&$filesWithLicenses, $treeTableName)
Map findings to the files.
const AVAILABLE_OUTPUT_FORMATS
Output formats available.
writeReport(&$packageNodes, $packageIds, $uploadId)
Write the report the file and update report table.
if(!defined('ENT_SUBSTITUTE')) convertToUTF8($content, $toHTML=true)
const OUTPUT_FORMAT_KEY
Argument key for output format.
addClearingStatus(&$filesWithLicenses, ItemTreeBounds $itemTreeBounds)
Add clearing status to the files.
computeUri($uploadId)
For a given upload, compute the URI and filename for the report.
static implodeLicenses($licenses, $spdxValidityChecker=null)
Implode licenses with "AND" or "OR".
char * trim(char *ptext)
Trimming whitespace.
materialize()
create temp table
toLicensesWithFilesAdder(&$filesWithLicenses, $licenses, $copyrights, $file, $fullPath)
Map licenses, copyrights, files and full path to filesWithLicenses array.