Skip to content

Commit

Permalink
Introduce JSON P3A pings that duplicate the legacy pings.
Browse files Browse the repository at this point in the history
  • Loading branch information
iefremov committed Dec 14, 2021
1 parent f83f6a2 commit 7896d08
Show file tree
Hide file tree
Showing 9 changed files with 267 additions and 9 deletions.
2 changes: 2 additions & 0 deletions components/p3a/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ static_library("p3a") {
"brave_p3a_service.cc",
"brave_p3a_service.h",
"brave_p3a_switches.h",
"brave_p3a_new_uploader.cc",
"brave_p3a_new_uploader.h",
"brave_p3a_uploader.cc",
"brave_p3a_uploader.h",
"brave_p3a_utils.h",
Expand Down
10 changes: 9 additions & 1 deletion components/p3a/brave_p3a_log_store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,15 @@ const std::string& BraveP3ALogStore::staged_log() const {
auto iter = log_.find(staged_entry_key_);
DCHECK(iter != log_.end());

return staged_log_;
return staged_log_.legacy_log;
}

const std::string& BraveP3ALogStore::staged_json_log() const {
DCHECK(!staged_entry_key_.empty());
auto iter = log_.find(staged_entry_key_);
DCHECK(iter != log_.end());

return staged_log_.json_log;
}

std::string BraveP3ALogStore::staged_log_type() const {
Expand Down
17 changes: 15 additions & 2 deletions components/p3a/brave_p3a_log_store.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,21 @@ namespace brave {
// for now persisted entries never expire.
class BraveP3ALogStore : public metrics::LogStore {
public:
struct LogForJsonMigration {
std::string legacy_log;
std::string json_log;

void clear() {
legacy_log.clear();
json_log.clear();
}
};


class Delegate {
public:
// Prepares a string representaion of an entry.
virtual std::string Serialize(base::StringPiece histogram_name,
virtual LogForJsonMigration Serialize(base::StringPiece histogram_name,
uint64_t value) = 0;
// Returns false if the metric is obsolete and should be cleaned up.
virtual bool IsActualMetric(base::StringPiece histogram_name) const = 0;
Expand All @@ -53,6 +64,8 @@ class BraveP3ALogStore : public metrics::LogStore {
bool has_unsent_logs() const override;
bool has_staged_log() const override;
const std::string& staged_log() const override;
// This will replace `staged_log` once we migrate to JSON pings.
const std::string& staged_json_log() const;
std::string staged_log_type() const;
const std::string& staged_log_hash() const override;
const std::string& staged_log_signature() const override;
Expand Down Expand Up @@ -93,7 +106,7 @@ class BraveP3ALogStore : public metrics::LogStore {
base::flat_set<std::string> unsent_entries_;

std::string staged_entry_key_;
std::string staged_log_;
LogForJsonMigration staged_log_;

// Not used for now.
std::string staged_log_hash_;
Expand Down
114 changes: 114 additions & 0 deletions components/p3a/brave_p3a_new_uploader.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/* Copyright 2021 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "brave/components/p3a/brave_p3a_new_uploader.h"

#include <utility>

#include "base/base64.h"
#include "net/base/load_flags.h"
#include "services/network/public/cpp/resource_request.h"
#include "services/network/public/cpp/shared_url_loader_factory.h"
#include "services/network/public/cpp/simple_url_loader.h"
#include "services/network/public/mojom/url_response_head.mojom.h"

namespace brave {

namespace {

// TODO(iefremov): Provide more details for the traffic annotation.
net::NetworkTrafficAnnotationTag GetNetworkTrafficAnnotation(
base::StringPiece upload_type) {
if (upload_type == "p3a") {
return net::DefineNetworkTrafficAnnotation("metrics_report_uma", R"(
semantics {
sender: "Brave Privacy-Preserving Product Analytics Uploader"
description:
"Report of anonymized usage statistics. For more info, see "
"https://brave.com/P3A"
trigger:
"Reports are automatically generated on startup and at intervals "
"while Brave is running."
data:
"A protocol buffer with anonymized and encrypted usage data."
destination: WEBSITE
}
policy {
cookies_allowed: NO
setting:
"Users can enable or disable it in brave://settings/privacy"
policy_exception_justification:
"Not implemented."
})");
}
DCHECK_EQ(upload_type, "p2a");
return net::DefineNetworkTrafficAnnotation("metrics_report_uma", R"(
semantics {
sender: "Brave Privacy-Preserving Ad Analytics Uploader"
description:
"Report of anonymized usage statistics. For more info, see "
"https://brave.com/P2A"
trigger:
"Reports are automatically generated on startup and at intervals "
"while Brave is running."
data:
"A protocol buffer with anonymized and encrypted usage data."
destination: WEBSITE
}
policy {
cookies_allowed: NO
setting:
"Users can enable or disable it by enabling or disabling Brave rewards
or ads in brave://rewards"
policy_exception_justification:
"Not implemented."
})");
}

} // namespace

BraveP3ANewUploader::BraveP3ANewUploader(
scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
const GURL& p3a_endpoint,
const GURL& p2a_endpoint)
: url_loader_factory_(url_loader_factory),
p3a_endpoint_(p3a_endpoint),
p2a_endpoint_(p2a_endpoint) {}

BraveP3ANewUploader::~BraveP3ANewUploader() = default;

void BraveP3ANewUploader::UploadLog(const std::string& compressed_log_data,
const std::string& upload_type) {
auto resource_request = std::make_unique<network::ResourceRequest>();
if (upload_type == "p2a") {
resource_request->url = p2a_endpoint_;
resource_request->headers.SetHeader("X-Brave-P2A", "?1");
} else if (upload_type == "p3a") {
resource_request->url = p3a_endpoint_;
resource_request->headers.SetHeader("X-Brave-P3A", "?1");
} else {
NOTREACHED();
}

resource_request->credentials_mode = network::mojom::CredentialsMode::kOmit;
resource_request->method = "POST";

url_loader_ = network::SimpleURLLoader::Create(
std::move(resource_request),
GetNetworkTrafficAnnotation(upload_type));
url_loader_->AttachStringForUpload(compressed_log_data, "application/json");

url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie(
url_loader_factory_.get(),
base::BindOnce(&BraveP3ANewUploader::OnUploadComplete,
base::Unretained(this)));
}

void BraveP3ANewUploader::OnUploadComplete(
std::unique_ptr<std::string> response_body) {
url_loader_.reset();
}

} // namespace brave
53 changes: 53 additions & 0 deletions components/p3a/brave_p3a_new_uploader.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/* Copyright 2021 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef BRAVE_COMPONENTS_P3A_BRAVE_P3A_NEW_UPLOADER_H_
#define BRAVE_COMPONENTS_P3A_BRAVE_P3A_NEW_UPLOADER_H_

#include <memory>
#include <string>

#include "base/memory/ref_counted.h"
#include "url/gurl.h"

namespace network {
class SharedURLLoaderFactory;
class SimpleURLLoader;
} // namespace network

namespace brave {

// This will replace the "normal" uploader when the server-side is ready.
// The difference is only in endpoint, mime and lack of base64 encoding.
// Also this one doesn't report the upload status back (since this one is for
// testing the new endpoint).
class BraveP3ANewUploader {
public:
BraveP3ANewUploader(
scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
const GURL& p3a_endpoint,
const GURL& p2a_endpoint);

BraveP3ANewUploader(const BraveP3ANewUploader&) = delete;
BraveP3ANewUploader& operator=(const BraveP3ANewUploader&) = delete;

~BraveP3ANewUploader();

// From metrics::MetricsLogUploader
void UploadLog(const std::string& compressed_log_data,
const std::string& upload_type);

void OnUploadComplete(std::unique_ptr<std::string> response_body);

private:
scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_;
const GURL p3a_endpoint_;
const GURL p2a_endpoint_;
std::unique_ptr<network::SimpleURLLoader> url_loader_;
};

} // namespace brave

#endif // BRAVE_COMPONENTS_P3A_BRAVE_P3A_NEW_UPLOADER_H_
33 changes: 29 additions & 4 deletions components/p3a/brave_p3a_service.cc
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

#include "base/command_line.h"
#include "base/i18n/timezone.h"
#include "base/json/json_writer.h"
#include "base/metrics/histogram_macros.h"
#include "base/metrics/histogram_samples.h"
#include "base/metrics/metrics_hashes.h"
Expand All @@ -28,6 +29,7 @@
#include "brave/components/p3a/brave_p3a_log_store.h"
#include "brave/components/p3a/brave_p3a_scheduler.h"
#include "brave/components/p3a/brave_p3a_switches.h"
#include "brave/components/p3a/brave_p3a_new_uploader.h"
#include "brave/components/p3a/brave_p3a_uploader.h"
#include "brave/components/p3a/pref_names.h"
#include "brave/components/version_info/version_info.h"
Expand All @@ -53,6 +55,9 @@ constexpr char kLastRotationTimeStampPref[] = "p3a.last_rotation_timestamp";
constexpr char kP3AServerUrl[] = "https://p3a.brave.com/";
constexpr char kP2AServerUrl[] = "https://p2a.brave.com/";

constexpr char kP3AJsonServerUrl[] = "https://p3a-json.brave.com/";
constexpr char kP2AJsonServerUrl[] = "https://p2a-json.brave.com/";

constexpr uint64_t kDefaultUploadIntervalSeconds = 60; // 1 minute.

// TODO(iefremov): Provide moar histograms!
Expand Down Expand Up @@ -280,6 +285,9 @@ void BraveP3AService::Init(
url_loader_factory, upload_server_url_, GURL(kP2AServerUrl),
base::BindRepeating(&BraveP3AService::OnLogUploadComplete, this)));

new_uploader_.reset(new BraveP3ANewUploader(
url_loader_factory, GURL(kP3AJsonServerUrl), GURL(kP2AJsonServerUrl)));

upload_scheduler_.reset(new BraveP3AScheduler(
base::BindRepeating(&BraveP3AService::StartScheduledUpload, this),
(randomize_upload_interval_
Expand All @@ -294,8 +302,9 @@ void BraveP3AService::Init(
}
}

std::string BraveP3AService::Serialize(base::StringPiece histogram_name,
uint64_t value) {
BraveP3ALogStore::LogForJsonMigration
BraveP3AService::Serialize(base::StringPiece histogram_name,
uint64_t value) {
// TRACE_EVENT0("brave_p3a", "SerializeMessage");
// TODO(iefremov): Maybe we should store it in logs and pass here?
// We cannot directly query |base::StatisticsRecorder::FindHistogram| because
Expand All @@ -306,7 +315,18 @@ std::string BraveP3AService::Serialize(base::StringPiece histogram_name,
UpdateMessageMeta();
brave_pyxis::RawP3AValue message;
GenerateP3AMessage(histogram_name_hash, value, message_meta_, &message);
return message.SerializeAsString();

base::Value p3a_json_value = GenerateP3AJsonMessage(histogram_name_hash,
value,
message_meta_);
std::string p3a_json_message;
const bool ok = base::JSONWriter::Write(p3a_json_value, &p3a_json_message);
DCHECK(ok);

return {
message.SerializeAsString(),
p3a_json_message
};
}

bool
Expand Down Expand Up @@ -407,6 +427,11 @@ void BraveP3AService::StartScheduledUpload() {
VLOG(2) << "StartScheduledUpload - Uploading " << log.size() << " bytes "
<< "of type " << log_type;
uploader_->UploadLog(log, log_type);

// We duplicate the uploads to test that the new approach works as fine
// as the legacy one. Once we are ready, we will remove the old
// uploader.
new_uploader_->UploadLog(log_store_->staged_json_log(), log_type);
}
}

Expand Down Expand Up @@ -512,7 +537,7 @@ void BraveP3AService::DoRotation() {
void BraveP3AService::UpdateRotationTimer() {
base::Time now = base::Time::Now();
base::Time next_rotation = rotation_interval_.is_zero()
? NextMonday(base::Time::Now())
? NextMonday(now)
: now + rotation_interval_;
if (now >= next_rotation) {
// Should never happen, but let's stay on the safe side.
Expand Down
8 changes: 6 additions & 2 deletions components/p3a/brave_p3a_service.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ namespace brave {

class BraveP3AScheduler;
class BraveP3AUploader;
class BraveP3ANewUploader;

// Core class for Brave Privacy-Preserving Product Analytics machinery.
// Works on UI thread. Refcounted to receive histogram updating callbacks
Expand All @@ -52,8 +53,9 @@ class BraveP3AService : public base::RefCountedThreadSafe<BraveP3AService>,
scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory);

// BraveP3ALogStore::Delegate
std::string Serialize(base::StringPiece histogram_name,
uint64_t value) override;
BraveP3ALogStore::LogForJsonMigration Serialize(
base::StringPiece histogram_name,
uint64_t value) override;

// May be accessed from multiple threads, so this is thread-safe.
bool IsActualMetric(base::StringPiece histogram_name) const override;
Expand Down Expand Up @@ -110,6 +112,8 @@ class BraveP3AService : public base::RefCountedThreadSafe<BraveP3AService>,
// Components:
std::unique_ptr<BraveP3ALogStore> log_store_;
std::unique_ptr<BraveP3AUploader> uploader_;
// See `brave_p3a_new_uploader.h`
std::unique_ptr<BraveP3ANewUploader> new_uploader_;
std::unique_ptr<BraveP3AScheduler> upload_scheduler_;

// Used to store histogram values that are produced between constructing
Expand Down
31 changes: 31 additions & 0 deletions components/p3a/p3a_message.cc
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,37 @@ void GenerateP3AMessage(uint64_t metric_hash,
p3a_message->set_p3a_info(data, kDataLength);
}

base::Value GenerateP3AJsonMessage(uint64_t metric_hash,
uint64_t metric_value,
const MessageMetainfo& meta) {
base::Value result(base::Value::Type::DICTIONARY);

// Find out years of install and survey.
base::Time::Exploded exploded;
meta.date_of_survey.LocalExplode(&exploded);
DCHECK_GE(exploded.year, 999);
result.SetIntKey("yos", exploded.year);

meta.date_of_install.LocalExplode(&exploded);
DCHECK_GE(exploded.year, 999);
result.SetIntKey("yoi", exploded.year);

// Fill meta.
result.SetStringKey("country_code", meta.country_code);
result.SetStringKey("platform", meta.platform);
result.SetStringKey("version", meta.version);
result.SetStringKey("channel", meta.channel);
result.SetIntKey("woi", meta.woi);
result.SetIntKey("wos", meta.wos);
result.SetStringKey("refcode", meta.refcode);

// Set the metric
result.SetStringKey("metric_hash", base::NumberToString(metric_hash));
result.SetIntKey("metric_value", metric_value);

return result;
}

void MaybeStripRefcodeAndCountry(MessageMetainfo* meta) {
const std::string& refcode = meta->refcode;
const std::string& country = meta->country_code;
Expand Down
Loading

0 comments on commit 7896d08

Please sign in to comment.