From 9ec21404db646bb49df2d905075fb697c84cd67e Mon Sep 17 00:00:00 2001 From: "James C. Owens" Date: Tue, 7 Jan 2025 11:29:29 -0500 Subject: [PATCH] Adjust scraper to collect total credit across all cpids This commit adds the internal machinery to the scraper to compute the total credit across ALL cpids for each project, regardless of whether they are active beaconholders or not. This is to support auto greylisting. This commit does NOT include the network manifest changes. --- src/gridcoin/scraper/scraper.cpp | 66 +++++++++++++++++++++++++------- src/gridcoin/scraper/scraper.h | 1 + 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/src/gridcoin/scraper/scraper.cpp b/src/gridcoin/scraper/scraper.cpp index 7bbc99bf69..a95d116f94 100755 --- a/src/gridcoin/scraper/scraper.cpp +++ b/src/gridcoin/scraper/scraper.cpp @@ -372,9 +372,10 @@ bool MarkScraperFileManifestEntryNonCurrent(ScraperFileManifestEntry& entry); * @param filetype * @param sProject * @param excludefromcsmanifest + * @param all_cpid_total_credit */ void AlignScraperFileManifestEntries(const fs::path& file, const std::string& filetype, const std::string& sProject, - const bool& excludefromcsmanifest); + const bool& excludefromcsmanifest, const double& all_cpid_total_credit); /** * @brief Constructs the scraper statistics from the current state of the scraper, which is all of the in scope files at the * time the function is called @@ -527,11 +528,12 @@ bool DownloadProjectRacFilesByCPID(const WhitelistSnapshot& projectWhitelist); * @param Consensus * @param GlobalVerifiedBeaconsCopy * @param IncomingVerifiedBeacons + * @param all_cpid_total_credit * @return bool true if successful */ bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& file, const std::string& etag, BeaconConsensus& Consensus, ScraperVerifiedBeacons& GlobalVerifiedBeaconsCopy, - ScraperVerifiedBeacons& IncomingVerifiedBeacons); + ScraperVerifiedBeacons& IncomingVerifiedBeacons, double& all_cpid_total_credit); /** * @brief Clears the authentication ETag auth.dat file */ @@ -2111,7 +2113,7 @@ bool DownloadProjectHostFiles(const WhitelistSnapshot& projectWhitelist) } // Save host xml files to file manifest map with exclude from CSManifest flag set to true. - AlignScraperFileManifestEntries(host_file, "host", prjs.m_name, true); + AlignScraperFileManifestEntries(host_file, "host", prjs.m_name, true, 0); } return true; @@ -2285,7 +2287,7 @@ bool DownloadProjectTeamFiles(const WhitelistSnapshot& projectWhitelist) // If in explorer mode and new file downloaded, save team xml files to file manifest map with exclude from CSManifest // flag set to true. If not in explorer mode, this is not necessary, because the team xml file is just temporary and // can be discarded after processing. - if (explorer_mode() && bDownloadFlag) AlignScraperFileManifestEntries(team_file, "team", prjs.m_name, true); + if (explorer_mode() && bDownloadFlag) AlignScraperFileManifestEntries(team_file, "team", prjs.m_name, true, 0); // If require team whitelist is set and bETagChanged is true, then process the file. This also populates/updated the // team whitelist TeamIDs in the TeamIDMap and the ETag entries in the ProjTeamETags map. @@ -2561,13 +2563,15 @@ bool DownloadProjectRacFilesByCPID(const WhitelistSnapshot& projectWhitelist) continue; } - // If in explorer mode, save user (rac) source xml files to file manifest map with exclude from CSManifest flag set - // to true. - if (explorer_mode()) AlignScraperFileManifestEntries(rac_file, "user_source", prjs.m_name, true); + double all_cpid_total_credit = 0.0; // Now that the source file is handled, process the file. ProcessProjectRacFileByCPID(prjs.m_name, rac_file, sRacETag, Consensus, - GlobalVerifiedBeaconsCopy, IncomingVerifiedBeacons); + GlobalVerifiedBeaconsCopy, IncomingVerifiedBeacons, all_cpid_total_credit); + + // If in explorer mode, save user (rac) source xml files to file manifest map with exclude from CSManifest flag set + // to true. + if (explorer_mode()) AlignScraperFileManifestEntries(rac_file, "user_source", prjs.m_name, true, all_cpid_total_credit); } // for prjs : projectWhitelist // Get the global verified beacons and copy the incoming verified beacons from the @@ -2615,7 +2619,7 @@ bool DownloadProjectRacFilesByCPID(const WhitelistSnapshot& projectWhitelist) // This version uses a consensus beacon map (and teamid, if team filtering is specified by policy) to filter statistics. bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& file, const std::string& etag, BeaconConsensus& Consensus, ScraperVerifiedBeacons& GlobalVerifiedBeaconsCopy, - ScraperVerifiedBeacons& IncomingVerifiedBeacons) + ScraperVerifiedBeacons& IncomingVerifiedBeacons, double& all_cpid_total_credit) { auto explorer_mode = []() { LOCK(cs_ScraperGlobals); return fExplorer; }; auto require_team_whitelist_membership = []() { LOCK(cs_ScraperGlobals); return REQUIRE_TEAM_WHITELIST_MEMBERSHIP; }; @@ -2721,6 +2725,16 @@ bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& fil } } + // We need to accumulate total credit across ALL project cpids, regardless of their beacon status, to get + // an "all cpid" total credit sum to be used for automatic greylisting purposes. + std::string s_cpid_total_credit = ExtractXML(data, "", ""); + double cpid_total_credit = 0; + + if (!ParseDouble(s_cpid_total_credit, &cpid_total_credit)) { + _log(logattribute::ERR, __func__, "Bad cpid total credit in user stats file data."); + continue; + } + // We do NOT want to add a just verified CPID to the statistics this iteration, if it was // not already active, because we may be halfway through processing the set of projects. // Instead, add to the incoming verification map (above), which will be handled in the @@ -2764,7 +2778,7 @@ bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& fil } // User beacon verified. Append its statistics to the CSV output. - out << ExtractXML(data, "", "") << "," + out << s_cpid_total_credit << "," << ExtractXML(data, "", "") << "," << ExtractXML(data, "", "") << "," << cpid @@ -2835,7 +2849,7 @@ bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& fil // Here, regardless of explorer mode, save processed rac files to file manifest map with exclude from CSManifest flag // set to false. - AlignScraperFileManifestEntries(gzetagfile, "user", project, false); + AlignScraperFileManifestEntries(gzetagfile, "user", project, false, all_cpid_total_credit); _log(logattribute::INFO, "ProcessProjectRacFileByCPID", "Complete Process"); @@ -3221,7 +3235,8 @@ EXCLUSIVE_LOCKS_REQUIRED(cs_StructScraperFileManifest) } void AlignScraperFileManifestEntries(const fs::path& file, const std::string& filetype, - const std::string& sProject, const bool& excludefromcsmanifest) + const std::string& sProject, const bool& excludefromcsmanifest, + const double& all_cpid_total_credit) { ScraperFileManifestEntry NewRecord; @@ -3238,6 +3253,7 @@ void AlignScraperFileManifestEntries(const fs::path& file, const std::string& fi NewRecord.current = true; NewRecord.excludefromcsmanifest = excludefromcsmanifest; NewRecord.filetype = filetype; + NewRecord.all_cpid_total_credit = all_cpid_total_credit; // Code block to lock StructScraperFileManifest during record insertion and delete because we want this atomic. { @@ -3381,6 +3397,27 @@ bool LoadScraperFileManifest(const fs::path& file) LoadEntry.filetype = "user"; } + // This handles startup with legacy manifest file without the all_cpid_total_credit column. + if (vline.size() >= 8) { + // In scraper for superblock v3 and autogreylist, we have to record total credit across all cpids, regardless + // of whether they are active beaconholders to support auto greylisting. + + double all_cpid_total_credit = 0.0; + + if (!ParseDouble(vline[7], &all_cpid_total_credit)) { + // This shouldn't happen given the conditional above, but to be thorough... + _log(logattribute::ERR, __func__, "The \"all_cpid_total_credit\" field not parsed correctly for a manifest " + "entry. Skipping."); + continue; + } + + LoadEntry.all_cpid_total_credit = all_cpid_total_credit; + } else { + // The default if the field is not there is user. (Because scraper ver 1 all files in the manifest are + // user.) + LoadEntry.filetype = "user"; + } + // Lock cs_StructScraperFileManifest before updating // global structure. { @@ -3425,6 +3462,7 @@ bool StoreScraperFileManifest(const fs::path& file) << "Filename," << "ExcludeFromCSManifest," << "Filetype" + << "All_cpid_total_credit" << "\n"; for (auto const& entry : StructScraperFileManifest.mScraperFileManifest) @@ -3437,7 +3475,9 @@ bool StoreScraperFileManifest(const fs::path& file) + entry.second.project + "," + entry.first + "," + ToString(entry.second.excludefromcsmanifest) + "," - + entry.second.filetype + "\n"; + + entry.second.filetype + + ToString(entry.second.all_cpid_total_credit) + + "\n"; stream << sScraperFileManifestEntry; } } diff --git a/src/gridcoin/scraper/scraper.h b/src/gridcoin/scraper/scraper.h index 2340e17b21..dc5e9b2637 100644 --- a/src/gridcoin/scraper/scraper.h +++ b/src/gridcoin/scraper/scraper.h @@ -80,6 +80,7 @@ struct ScraperFileManifestEntry bool current = true; bool excludefromcsmanifest = true; std::string filetype; + double all_cpid_total_credit = 0; }; /**