Commit f1a0d5d8 authored by kundaji's avatar kundaji Committed by Commit bot

Create a DataUseRecorder instance for each page load in Chrome.

Map each URLRequest originating from a frame to its DataUseRecorder.
Create a new DataUseRecorder for each URLRequest from Chrome services.
Move DataUseRecorders from pending navigation map to render frame
map when navigation commits.

BUG=663532

Review-Url: https://codereview.chromium.org/2534023002
Cr-Commit-Position: refs/heads/master@{#435544}
parent 6a32e4a0
......@@ -288,6 +288,8 @@ split_static_library("browser") {
"data_use_measurement/chrome_data_use_ascriber_service.h",
"data_use_measurement/chrome_data_use_ascriber_service_factory.cc",
"data_use_measurement/chrome_data_use_ascriber_service_factory.h",
"data_use_measurement/chrome_data_use_recorder.cc",
"data_use_measurement/chrome_data_use_recorder.h",
"data_use_measurement/data_use_web_contents_observer.cc",
"data_use_measurement/data_use_web_contents_observer.h",
"defaults.cc",
......
......@@ -7,6 +7,8 @@
#include <list>
#include <memory>
#include <string>
#include <tuple>
#include <unordered_map>
#include <utility>
......@@ -14,6 +16,7 @@
#include "base/hash.h"
#include "base/macros.h"
#include "base/supports_user_data.h"
#include "chrome/browser/data_use_measurement/chrome_data_use_recorder.h"
#include "components/data_use_measurement/core/data_use_ascriber.h"
#include "content/public/browser/global_request_id.h"
#include "url/gurl.h"
......@@ -52,14 +55,10 @@ class ChromeDataUseAscriber : public DataUseAscriber {
~ChromeDataUseAscriber() override;
// DataUseAscriber implementation:
DataUseRecorder* GetDataUseRecorder(net::URLRequest* request) override;
// Called before a request is sent.
ChromeDataUseRecorder* GetDataUseRecorder(net::URLRequest* request,
bool can_create_new) override;
void OnBeforeUrlRequest(net::URLRequest* request) override;
// Called when a URLRequest is being destroyed.
void OnUrlRequestDestroyed(net::URLRequest* request) override;
std::unique_ptr<URLRequestClassifier> CreateURLRequestClassifier()
const override;
......@@ -91,21 +90,14 @@ class ChromeDataUseAscriber : public DataUseAscriber {
bool is_same_page_navigation,
void* navigation_handle);
// Called when a main frame navigation is redirected.
void DidRedirectMainFrameNavigation(GURL gurl,
int render_process_id,
int render_frame_id,
void* navigation_handle);
private:
// Use as a key in the render frame map. Corresponds to a unique
// RenderFrameHost.
typedef std::pair<int, int> RenderFrameHostID;
friend class ChromeDataUseAscriberTest;
// Entry in the |data_use_recorders_| list which owns all instances of
// DataUseRecorder.
typedef std::list<std::unique_ptr<data_use_measurement::DataUseRecorder>>::
iterator DataUseRecorderEntry;
typedef std::list<ChromeDataUseRecorder> DataUseRecorderList;
typedef DataUseRecorderList::iterator DataUseRecorderEntry;
struct GlobalRequestIDHash {
public:
......@@ -128,13 +120,22 @@ class ChromeDataUseAscriber : public DataUseAscriber {
DataUseRecorderEntry entry_;
};
void DeletePendingNavigationEntry(content::GlobalRequestID global_request_id);
DataUseRecorderEntry GetDataUseRecorderEntry(net::URLRequest* request,
bool can_create_new);
void OnDataUseCompleted(DataUseRecorderEntry entry);
DataUseRecorderEntry CreateNewDataUseRecorder(net::URLRequest* request);
bool IsRecorderInPendingNavigationMap(net::URLRequest* request);
bool IsRecorderInRenderFrameMap(net::URLRequest* request);
// Owner for all instances of DataUseRecorder. An instance is kept in this
// list if any entity (render frame hosts, URLRequests, pending navigations)
// that ascribe data use to the instance exists, and deleted when all
// ascribing entities go away.
std::list<std::unique_ptr<DataUseRecorder>> data_use_recorders_;
DataUseRecorderList data_use_recorders_;
// Map from RenderFrameHost to the DataUseRecorderEntry in
// |data_use_recorders_| that the frame ascribe data use to.
......
......@@ -115,13 +115,11 @@ void ChromeDataUseAscriberService::RenderFrameDeleted(
void ChromeDataUseAscriberService::DidStartNavigation(
content::NavigationHandle* navigation_handle) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
if (!navigation_handle->IsInMainFrame())
return;
if (!ascriber_)
return;
content::WebContents* web_contents = navigation_handle->GetWebContents();
content::BrowserThread::PostTask(
content::BrowserThread::IO, FROM_HERE,
......@@ -155,26 +153,6 @@ void ChromeDataUseAscriberService::ReadyToCommitNavigation(
navigation_handle));
}
void ChromeDataUseAscriberService::DidRedirectNavigation(
content::NavigationHandle* navigation_handle) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
if (!is_initialized_ || !navigation_handle->IsInMainFrame())
return;
if (!ascriber_)
return;
content::WebContents* web_contents = navigation_handle->GetWebContents();
content::BrowserThread::PostTask(
content::BrowserThread::IO, FROM_HERE,
base::Bind(&ChromeDataUseAscriber::DidRedirectMainFrameNavigation,
base::Unretained(ascriber_), navigation_handle->GetURL(),
web_contents->GetRenderProcessHost()->GetID(),
web_contents->GetMainFrame()->GetRoutingID(),
navigation_handle));
}
void ChromeDataUseAscriberService::SetDataUseAscriber(
ChromeDataUseAscriber* ascriber) {
DCHECK(!is_initialized_);
......
......@@ -53,11 +53,6 @@ class ChromeDataUseAscriberService : public KeyedService {
// methods cannot be called on the IO thread, so the pointer is cast to void*.
void ReadyToCommitNavigation(content::NavigationHandle* navigation_handle);
// Called when a navigation is redirected. Propagates main frame navigation
// redirect to the |ascriber_| on the IO thread. NavigationHandle methods
// cannot be called on the IO thread, so the pointer is cast to void*.
void DidRedirectNavigation(content::NavigationHandle* navigation_handle);
private:
friend class ChromeDataUseAscriberServiceTest;
......
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/data_use_measurement/chrome_data_use_ascriber.h"
#include <list>
#include <memory>
#include "base/message_loop/message_loop.h"
#include "base/run_loop.h"
#include "components/data_use_measurement/core/data_use_recorder.h"
#include "content/public/browser/resource_request_info.h"
#include "content/public/common/browser_side_navigation_policy.h"
#include "content/public/common/process_type.h"
#include "content/public/test/mock_resource_context.h"
#include "content/public/test/test_browser_thread_bundle.h"
#include "net/url_request/url_request_test_util.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace {
int kRenderProcessId = 1;
int kRenderFrameId = 2;
int kRequestId = 3;
}
namespace data_use_measurement {
class ChromeDataUseAscriberTest : public testing::Test {
protected:
ChromeDataUseAscriberTest()
: thread_bundle_(content::TestBrowserThreadBundle::IO_MAINLOOP),
resource_context_(new content::MockResourceContext(&context_)) {}
void SetUp() override {}
void TearDown() override { recorders().clear(); }
std::list<ChromeDataUseRecorder>& recorders() {
return ascriber_.data_use_recorders_;
}
net::TestURLRequestContext* context() { return &context_; }
content::MockResourceContext* resource_context() {
return resource_context_.get();
}
ChromeDataUseAscriber* ascriber() { return &ascriber_; }
std::unique_ptr<net::URLRequest> CreateNewRequest(std::string url,
bool is_main_frame,
int request_id,
int render_process_id,
int render_frame_id) {
std::unique_ptr<net::URLRequest> request =
context()->CreateRequest(GURL(url), net::IDLE, nullptr);
// TODO(kundaji): Allow request_id to be specified in AllocateForTesting.
content::ResourceRequestInfo::AllocateForTesting(
request.get(),
content::RESOURCE_TYPE_MAIN_FRAME,
resource_context(),
render_process_id,
-1, // render_view_id
render_frame_id,
is_main_frame,
false, // parent_is_main_frame
false, // allow_download
true, // is_async
false); // is_using_lofi
return request;
}
private:
content::TestBrowserThreadBundle thread_bundle_;
ChromeDataUseAscriber ascriber_;
net::TestURLRequestContext context_;
std::unique_ptr<content::MockResourceContext> resource_context_;
};
TEST_F(ChromeDataUseAscriberTest, NoRecorderWithoutFrame) {
if (content::IsBrowserSideNavigationEnabled())
return;
std::unique_ptr<net::URLRequest> request = CreateNewRequest(
"http://test.com", true, kRequestId, kRenderProcessId, kRenderFrameId);
// Main frame request should not cause a recorder to be created, since the
// frame does not exist.
ascriber()->OnBeforeUrlRequest(request.get());
EXPECT_EQ(0u, recorders().size());
// Frame is created.
ascriber()->RenderFrameCreated(kRenderProcessId, kRenderFrameId, -1, -1);
EXPECT_EQ(1u, recorders().size());
// Request should cause a recorder to be created.
ascriber()->OnBeforeUrlRequest(request.get());
EXPECT_EQ(2u, recorders().size());
}
} // namespace data_use_measurement
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/data_use_measurement/chrome_data_use_recorder.h"
namespace data_use_measurement {
ChromeDataUseRecorder::ChromeDataUseRecorder() : main_frame_id_(-1, -1) {}
ChromeDataUseRecorder::~ChromeDataUseRecorder() {}
} // namespace data_use_measurement
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef CHROME_BROWSER_DATA_USE_MEASUREMENT_CHROME_DATA_USE_RECORDER_H_
#define CHROME_BROWSER_DATA_USE_MEASUREMENT_CHROME_DATA_USE_RECORDER_H_
#include <utility>
#include "base/macros.h"
#include "components/data_use_measurement/core/data_use_recorder.h"
#include "content/public/browser/global_request_id.h"
namespace data_use_measurement {
typedef std::pair<int, int> RenderFrameHostID;
class ChromeDataUseRecorder : public DataUseRecorder {
public:
ChromeDataUseRecorder();
~ChromeDataUseRecorder();
RenderFrameHostID main_frame_id() const { return main_frame_id_; }
void set_main_frame_id(RenderFrameHostID frame_id) {
main_frame_id_ = frame_id;
}
content::GlobalRequestID main_frame_request_id() const {
return main_frame_request_id_;
}
void set_main_frame_request_id(content::GlobalRequestID request_id) {
main_frame_request_id_ = request_id;
}
private:
// Identifier for the main frame for the page load this recorder is tracking.
// Only valid if the data use is associated with a page load.
RenderFrameHostID main_frame_id_;
// Identifier for the MAIN_FRAME request for this page load. Only valid if
// the data use is associated with a page load.
content::GlobalRequestID main_frame_request_id_;
DISALLOW_COPY_AND_ASSIGN(ChromeDataUseRecorder);
};
} // namespace data_use_measurement
#endif // CHROME_BROWSER_DATA_USE_MEASUREMENT_CHROME_DATA_USE_RECORDER_H_
......@@ -72,9 +72,4 @@ void DataUseWebContentsObserver::ReadyToCommitNavigation(
service_->ReadyToCommitNavigation(navigation_handle);
}
void DataUseWebContentsObserver::DidRedirectNavigation(
content::NavigationHandle* navigation_handle) {
service_->DidRedirectNavigation(navigation_handle);
}
} // namespace data_use_measurement
......@@ -39,12 +39,9 @@ class DataUseWebContentsObserver
content::NavigationHandle* navigation_handle) override;
void ReadyToCommitNavigation(
content::NavigationHandle* navigation_handle) override;
void DidRedirectNavigation(
content::NavigationHandle* navigation_handle) override;
private:
friend class content::WebContentsUserData<DataUseWebContentsObserver>;
DataUseWebContentsObserver(content::WebContents* web_contents,
ChromeDataUseAscriberService* service);
ChromeDataUseAscriberService* const service_;
......
......@@ -3121,6 +3121,7 @@ test("unit_tests") {
"../browser/data_usage/tab_id_annotator_unittest.cc",
"../browser/data_usage/tab_id_provider_unittest.cc",
"../browser/data_use_measurement/chrome_data_use_ascriber_service_unittest.cc",
"../browser/data_use_measurement/chrome_data_use_ascriber_unittest.cc",
"../browser/devtools/devtools_network_controller_unittest.cc",
"../browser/download/all_download_item_notifier_unittest.cc",
"../browser/download/chrome_download_manager_delegate_unittest.cc",
......
......@@ -8,6 +8,10 @@ namespace data_use_measurement {
DataUse::DataUse() : total_bytes_sent_(0), total_bytes_received_(0) {}
DataUse::DataUse(const DataUse& other) :
total_bytes_sent_(other.total_bytes_sent_),
total_bytes_received_(other.total_bytes_received_) {}
DataUse::~DataUse() {}
void DataUse::MergeFrom(const DataUse& other) {
......
......@@ -19,6 +19,7 @@ namespace data_use_measurement {
class DataUse {
public:
DataUse();
DataUse(const DataUse& other);
~DataUse();
// Merge data use from another instance.
......@@ -46,8 +47,6 @@ class DataUse {
int64_t total_bytes_sent_;
int64_t total_bytes_received_;
DISALLOW_COPY_AND_ASSIGN(DataUse);
};
} // namespace data_use_measurement
......
......@@ -22,31 +22,30 @@ std::unique_ptr<net::NetworkDelegate> DataUseAscriber::CreateNetworkDelegate(
}
void DataUseAscriber::OnBeforeUrlRequest(net::URLRequest* request) {
DataUseRecorder* recorder = GetDataUseRecorder(request);
DataUseRecorder* recorder = GetDataUseRecorder(request, true);
if (recorder)
recorder->OnBeforeUrlRequest(request);
}
void DataUseAscriber::OnBeforeRedirect(net::URLRequest* request,
const GURL& new_location) {}
void DataUseAscriber::OnNetworkBytesSent(net::URLRequest* request,
int64_t bytes_sent) {
DataUseRecorder* recorder = GetDataUseRecorder(request);
DataUseRecorder* recorder = GetDataUseRecorder(request, false);
if (recorder)
recorder->OnNetworkBytesSent(request, bytes_sent);
}
void DataUseAscriber::OnNetworkBytesReceived(net::URLRequest* request,
int64_t bytes_received) {
DataUseRecorder* recorder = GetDataUseRecorder(request);
DataUseRecorder* recorder = GetDataUseRecorder(request, false);
if (recorder)
recorder->OnNetworkBytesReceived(request, bytes_received);
}
void DataUseAscriber::OnUrlRequestCompleted(net::URLRequest* request,
bool started) {}
void DataUseAscriber::OnUrlRequestDestroyed(net::URLRequest* request) {
DataUseRecorder* recorder = GetDataUseRecorder(request);
// TODO(kundaji): Enforce DCHECK(recorder).
DataUseRecorder* recorder = GetDataUseRecorder(request, true);
if (recorder)
recorder->OnUrlRequestDestroyed(request);
}
......
......@@ -39,24 +39,22 @@ class DataUseAscriber {
// Returns the DataUseRecorder to which data usage for the given URL should
// be ascribed. If no existing DataUseRecorder exists, a new one will be
// created.
virtual DataUseRecorder* GetDataUseRecorder(net::URLRequest* request) = 0;
// created only if |can_create_new| is true.
virtual DataUseRecorder* GetDataUseRecorder(net::URLRequest* request,
bool can_create_new) = 0;
// Returns a URLRequestClassifier that can classify requests for metrics
// recording.
virtual std::unique_ptr<URLRequestClassifier> CreateURLRequestClassifier()
const = 0;
// Methods called by DataUseNetworkDelegate to propagate data use information:
virtual void OnBeforeUrlRequest(net::URLRequest* request);
virtual void OnBeforeRedirect(net::URLRequest* request,
const GURL& new_location);
virtual void OnNetworkBytesSent(net::URLRequest* request, int64_t bytes_sent);
virtual void OnNetworkBytesReceived(net::URLRequest* request,
int64_t bytes_received);
virtual void OnUrlRequestCompleted(net::URLRequest* request, bool started);
virtual void OnUrlRequestDestroyed(net::URLRequest* request);
virtual std::unique_ptr<URLRequestClassifier> CreateURLRequestClassifier()
const = 0;
};
} // namespace data_use_measurement
......
......@@ -37,7 +37,6 @@ void DataUseNetworkDelegate::OnBeforeURLRequestInternal(
void DataUseNetworkDelegate::OnBeforeRedirectInternal(
net::URLRequest* request,
const GURL& new_location) {
ascriber_->OnBeforeRedirect(request, new_location);
data_use_measurement_.OnBeforeRedirect(*request, new_location);
}
......@@ -55,14 +54,14 @@ void DataUseNetworkDelegate::OnNetworkBytesSentInternal(
data_use_measurement_.OnNetworkBytesSent(*request, bytes_sent);
}
void DataUseNetworkDelegate::OnURLRequestDestroyedInternal(
net::URLRequest* request) {
ascriber_->OnUrlRequestDestroyed(request);
}
void DataUseNetworkDelegate::OnCompletedInternal(net::URLRequest* request,
bool started) {
data_use_measurement_.OnCompleted(*request, started);
}
void DataUseNetworkDelegate::OnURLRequestDestroyedInternal(
net::URLRequest* request) {
ascriber_->OnUrlRequestDestroyed(request);
}
} // namespace data_use_measurement
......@@ -38,7 +38,8 @@ class TestDataUseAscriber : public DataUseAscriber {
public:
TestDataUseAscriber() {}
DataUseRecorder* GetDataUseRecorder(net::URLRequest* request) override {
DataUseRecorder* GetDataUseRecorder(net::URLRequest* request,
bool can_create_new) override {
return nullptr;
}
......
......@@ -8,7 +8,7 @@
namespace data_use_measurement {
DataUseRecorder::DataUseRecorder() {}
DataUseRecorder::DataUseRecorder() : main_url_request_(nullptr) {}
DataUseRecorder::~DataUseRecorder() {}
......@@ -16,7 +16,7 @@ bool DataUseRecorder::IsDataUseComplete() {
return pending_url_requests_.empty() && pending_data_sources_.empty();
}
void DataUseRecorder::OnBeforeUrlRequest(net::URLRequest* request) {
void DataUseRecorder::AddPendingURLRequest(net::URLRequest* request) {
pending_url_requests_.insert(request);
}
......@@ -24,6 +24,12 @@ void DataUseRecorder::OnUrlRequestDestroyed(net::URLRequest* request) {
pending_url_requests_.erase(request);
}
void DataUseRecorder::RemoveAllPendingURLRequests() {
pending_url_requests_.clear();
}
void DataUseRecorder::OnBeforeUrlRequest(net::URLRequest* request) {}
void DataUseRecorder::OnNetworkBytesReceived(net::URLRequest* request,
int64_t bytes_received) {
data_use_.total_bytes_received_ += bytes_received;
......@@ -46,8 +52,12 @@ void DataUseRecorder::RemovePendingDataSource(void* source) {
pending_data_sources_.erase(source);
}
bool DataUseRecorder::HasPendingURLRequest(const net::URLRequest* request) {
bool DataUseRecorder::HasPendingURLRequest(net::URLRequest* request) {
return pending_url_requests_.find(request) != pending_url_requests_.end();
}
void DataUseRecorder::MergeFrom(DataUseRecorder* other) {
data_use_.MergeFrom(other->data_use());
}
} // namespace data_use_measurement
......@@ -30,11 +30,37 @@ class DataUseRecorder {
// Returns the actual data used by the entity being tracked.
DataUse& data_use() { return data_use_; }
const base::hash_set<net::URLRequest*>& pending_url_requests() const {
return pending_url_requests_;
}
const net::URLRequest* main_url_request() const { return main_url_request_; }
void set_main_url_request(const net::URLRequest* request) {
main_url_request_ = request;
}
// Returns whether data use is complete and no additional data can be used
// by the entity tracked by this recorder. For example,
bool IsDataUseComplete();
// Adds |request| to the list of pending URLRequests that ascribe data use to
// this recorder.
void AddPendingURLRequest(net::URLRequest* request);
// Clears the list of pending URLRequests that ascribe data use to this
// recorder.
void RemoveAllPendingURLRequests();
// Returns whether there are any pending URLRequests whose data use is tracked
// by this DataUseRecorder.
bool HasPendingURLRequest(net::URLRequest* request);
// Merge another DataUseRecorder to this instance.
void MergeFrom(DataUseRecorder* other);
private:
friend class DataUseAscriber;
// Methods for tracking data use sources. These sources can initiate
// URLRequests directly or indirectly. The entity whose data use is being
// tracked by this recorder may comprise of sub-entities each of which use
......@@ -45,16 +71,6 @@ class DataUseRecorder {
bool HasPendingDataSource(void* source);
void RemovePendingDataSource(void* source);
// Returns whether there are any pending URLRequests whose data use is tracked
// by this DataUseRecorder.
bool HasPendingURLRequest(const net::URLRequest* request);
// Method to merge another DataUseRecorder to this instance.
void MergeWith(DataUseRecorder* other);
private:
friend class DataUseAscriber;
// Network Delegate methods:
void OnBeforeUrlRequest(net::URLRequest* request);
void OnUrlRequestDestroyed(net::URLRequest* request);
......@@ -62,12 +78,16 @@ class DataUseRecorder {
void OnNetworkBytesReceived(net::URLRequest* request, int64_t bytes_received);
// Pending URLRequests whose data is being tracked by this DataUseRecorder.
base::hash_set<const net::URLRequest*> pending_url_requests_;
base::hash_set<net::URLRequest*> pending_url_requests_;
// Data sources other than URLRequests, whose data is being tracked by this
// DataUseRecorder.
base::hash_set<const void*> pending_data_sources_;
// The main frame URLRequest for page loads. Null if this is not tracking a
// page load.
const net::URLRequest* main_url_request_;
// The network data use measured by this DataUseRecorder.
DataUse data_use_;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment