Commit f1a0d5d8 authored by kundaji's avatar kundaji Committed by Commit bot
Browse files

Create a DataUseRecorder instance for each page load in Chrome.

Map each URLRequest originating from a frame to its DataUseRecorder.
Create a new DataUseRecorder for each URLRequest from Chrome services.
Move DataUseRecorders from pending navigation map to render frame
map when navigation commits.

BUG=663532

Review-Url: https://codereview.chromium.org/2534023002
Cr-Commit-Position: refs/heads/master@{#435544}
parent 6a32e4a0
......@@ -288,6 +288,8 @@ split_static_library("browser") {
"data_use_measurement/chrome_data_use_ascriber_service.h",
"data_use_measurement/chrome_data_use_ascriber_service_factory.cc",
"data_use_measurement/chrome_data_use_ascriber_service_factory.h",
"data_use_measurement/chrome_data_use_recorder.cc",
"data_use_measurement/chrome_data_use_recorder.h",
"data_use_measurement/data_use_web_contents_observer.cc",
"data_use_measurement/data_use_web_contents_observer.h",
"defaults.cc",
......
......@@ -4,7 +4,10 @@
#include "chrome/browser/data_use_measurement/chrome_data_use_ascriber.h"
#include <string>
#include "base/memory/ptr_util.h"
#include "chrome/browser/data_use_measurement/chrome_data_use_recorder.h"
#include "components/data_use_measurement/content/content_url_request_classifier.h"
#include "components/data_use_measurement/core/data_use_recorder.h"
#include "components/data_use_measurement/core/data_use_user_data.h"
......@@ -14,6 +17,7 @@
#include "content/public/browser/render_frame_host.h"
#include "content/public/browser/resource_request_info.h"
#include "content/public/common/browser_side_navigation_policy.h"
#include "ipc/ipc_message.h"
#include "net/url_request/url_request.h"
namespace data_use_measurement {
......@@ -39,14 +43,100 @@ ChromeDataUseAscriber::~ChromeDataUseAscriber() {
DCHECK_EQ(0u, data_use_recorders_.size());
}
DataUseRecorder* ChromeDataUseAscriber::GetDataUseRecorder(
net::URLRequest* request) {
ChromeDataUseRecorder* ChromeDataUseAscriber::GetDataUseRecorder(
net::URLRequest* request,
bool can_create_new) {
DataUseRecorderEntry entry = GetDataUseRecorderEntry(request, can_create_new);
return entry == data_use_recorders_.end() ? nullptr : &(*entry);
}
ChromeDataUseAscriber::DataUseRecorderEntry
ChromeDataUseAscriber::GetDataUseRecorderEntry(net::URLRequest* request,
bool can_create_new) {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
return nullptr;
// TODO(ryansturm): Handle PlzNavigate (http://crbug/664233).
if (content::IsBrowserSideNavigationEnabled())
return data_use_recorders_.end();
// If a DataUseRecorder has already been set as user data, then return that.
auto user_data = static_cast<DataUseRecorderEntryAsUserData*>(
request->GetUserData(DataUseRecorderEntryAsUserData::kUserDataKey));
if (user_data)
return user_data->recorder_entry();
if (!can_create_new)
return data_use_recorders_.end();
// If request is associated with a ChromeService, create a new
// DataUseRecorder for it. There is no reason to aggregate URLRequests
// from ChromeServices into the same DataUseRecorder instance.
DataUseUserData* service = static_cast<DataUseUserData*>(
request->GetUserData(DataUseUserData::kUserDataKey));
if (service) {
DataUseRecorderEntry entry = CreateNewDataUseRecorder(request);
entry->data_use().set_description(
DataUseUserData::GetServiceNameAsString(service->service_name()));
return entry;
}
int render_process_id = -1;
int render_frame_id = -1;
bool has_valid_frame = content::ResourceRequestInfo::GetRenderFrameForRequest(
request, &render_process_id, &render_frame_id);
if (has_valid_frame &&
render_frame_id != SpecialRoutingIDs::MSG_ROUTING_NONE) {
DCHECK(render_process_id >= 0 || render_frame_id >= 0);
// Browser tests may not set up DataUseWebContentsObservers in which case
// this class never sees navigation and frame events so DataUseRecorders
// will never be destroyed. To avoid this, we ignore requests whose
// render frames don't have a record. However, this can also be caused by
// URLRequests racing the frame create events.
// TODO(kundaji): Add UMA.
RenderFrameHostID frame_key(render_process_id, render_frame_id);
auto frame_iter = render_frame_data_use_map_.find(frame_key);
if (frame_iter == render_frame_data_use_map_.end()) {
return data_use_recorders_.end();
}
const content::ResourceRequestInfo* request_info =
content::ResourceRequestInfo::ForRequest(request);
content::ResourceType resource_type =
request_info ? request_info->GetResourceType()
: content::RESOURCE_TYPE_LAST_TYPE;
if (resource_type == content::RESOURCE_TYPE_MAIN_FRAME) {
content::GlobalRequestID navigation_key =
request_info->GetGlobalRequestID();
DataUseRecorderEntry new_entry = CreateNewDataUseRecorder(request);
new_entry->set_main_frame_request_id(navigation_key);
pending_navigation_data_use_map_.insert(
std::make_pair(navigation_key, new_entry));
return new_entry;
}
DCHECK(frame_iter != render_frame_data_use_map_.end());
auto entry = frame_iter->second;
request->SetUserData(DataUseRecorderEntryAsUserData::kUserDataKey,
new DataUseRecorderEntryAsUserData(entry));
entry->AddPendingURLRequest(request);
return entry;
}
// Create a new DataUseRecorder for all other requests.
DataUseRecorderEntry entry = CreateNewDataUseRecorder(request);
DataUse& data_use = entry->data_use();
data_use.set_url(request->url());
return entry;
}
void ChromeDataUseAscriber::OnBeforeUrlRequest(net::URLRequest* request) {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
DataUseAscriber::OnBeforeUrlRequest(request);
// TODO(kundaji): Handle PlzNavigate.
if (content::IsBrowserSideNavigationEnabled())
......@@ -90,8 +180,8 @@ void ChromeDataUseAscriber::OnBeforeUrlRequest(net::URLRequest* request) {
if (user_data)
return;
DataUseRecorderEntry entry = data_use_recorders_.insert(
data_use_recorders_.end(), base::MakeUnique<DataUseRecorder>());
DataUseRecorderEntry entry = data_use_recorders_.emplace(
data_use_recorders_.end());
request->SetUserData(DataUseRecorderEntryAsUserData::kUserDataKey,
new DataUseRecorderEntryAsUserData(entry));
pending_navigation_data_use_map_.insert(
......@@ -99,18 +189,51 @@ void ChromeDataUseAscriber::OnBeforeUrlRequest(net::URLRequest* request) {
}
void ChromeDataUseAscriber::OnUrlRequestDestroyed(net::URLRequest* request) {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
DataUseRecorderEntry entry = GetDataUseRecorderEntry(request, true);
if (entry == data_use_recorders_.end())
return;
DataUseRecorder* recorder = &(*entry);
RenderFrameHostID frame_key = entry->main_frame_id();
auto frame_iter = render_frame_data_use_map_.find(frame_key);
bool is_in_render_frame_map =
frame_iter != render_frame_data_use_map_.end() &&
frame_iter->second->HasPendingURLRequest(request);
const content::ResourceRequestInfo* request_info =
content::ResourceRequestInfo::ForRequest(request);
content::ResourceType resource_type = request_info
? request_info->GetResourceType()
: content::RESOURCE_TYPE_LAST_TYPE;
if (resource_type == content::RESOURCE_TYPE_MAIN_FRAME) {
// If request was not successful, then ReadyToCommitNavigation will not be
// called. So delete the pending navigation DataUseRecorderEntry here.
if (!request->status().is_success()) {
DeletePendingNavigationEntry(request_info->GetGlobalRequestID());
bool is_in_pending_navigation_map = false;
if (request_info && resource_type == content::RESOURCE_TYPE_MAIN_FRAME) {
auto navigation_iter = pending_navigation_data_use_map_.find(
entry->main_frame_request_id());
is_in_pending_navigation_map =
navigation_iter != pending_navigation_data_use_map_.end();
// If request was not successful, then NavigationHandle in
// DidFinishMainFrameNavigation will not have GlobalRequestID. So we erase
// the DataUseRecorderEntry here.
if (is_in_pending_navigation_map && !request->status().is_success()) {
pending_navigation_data_use_map_.erase(navigation_iter);
is_in_pending_navigation_map = false;
}
}
DataUseAscriber::OnUrlRequestDestroyed(request);
request->RemoveUserData(DataUseRecorderEntryAsUserData::kUserDataKey);
if (recorder->IsDataUseComplete() && !is_in_render_frame_map &&
!is_in_pending_navigation_map) {
OnDataUseCompleted(entry);
data_use_recorders_.erase(entry);
}
}
void ChromeDataUseAscriber::RenderFrameCreated(int render_process_id,
......@@ -119,12 +242,30 @@ void ChromeDataUseAscriber::RenderFrameCreated(int render_process_id,
int parent_render_frame_id) {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
// TODO(kundaji): Point child render frames to the same DataUseRecorder as
// parent render frame.
DataUseRecorderEntry entry = data_use_recorders_.insert(
data_use_recorders_.end(), base::MakeUnique<DataUseRecorder>());
render_frame_data_use_map_.insert(std::make_pair(
RenderFrameHostID(render_process_id, render_frame_id), entry));
if (content::IsBrowserSideNavigationEnabled())
return;
if (parent_render_process_id != -1 && parent_render_frame_id != -1) {
// Create an entry in |render_frame_data_use_map_| for this frame with
// the same DataUseRecorderEntry as its parent frame.
auto frame_iter = render_frame_data_use_map_.find(
RenderFrameHostID(parent_render_process_id, parent_render_frame_id));
DCHECK(frame_iter != render_frame_data_use_map_.end());
DataUseRecorderEntry entry = frame_iter->second;
render_frame_data_use_map_.insert(std::make_pair(
RenderFrameHostID(render_process_id, render_frame_id), entry));
} else {
auto frame_iter = render_frame_data_use_map_.find(
RenderFrameHostID(render_process_id, render_frame_id));
DCHECK(frame_iter == render_frame_data_use_map_.end());
DataUseRecorderEntry entry = CreateNewDataUseRecorder(nullptr);
entry->set_main_frame_id(
RenderFrameHostID(render_process_id, render_frame_id));
render_frame_data_use_map_.insert(std::make_pair(
RenderFrameHostID(render_process_id, render_frame_id), entry));
}
}
void ChromeDataUseAscriber::RenderFrameDeleted(int render_process_id,
......@@ -132,13 +273,25 @@ void ChromeDataUseAscriber::RenderFrameDeleted(int render_process_id,
int parent_render_process_id,
int parent_render_frame_id) {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
if (content::IsBrowserSideNavigationEnabled())
return;
RenderFrameHostID key(render_process_id, render_frame_id);
auto frame_iter = render_frame_data_use_map_.find(key);
DCHECK(frame_iter != render_frame_data_use_map_.end());
DataUseRecorderEntry entry = frame_iter->second;
render_frame_data_use_map_.erase(frame_iter);
DataUseRecorder* recorder = &(*entry);
if (parent_render_process_id == -1 && parent_render_frame_id == -1 &&
recorder->IsDataUseComplete()) {
OnDataUseCompleted(entry);
data_use_recorders_.erase(entry);
}
data_use_recorders_.erase(entry);
render_frame_data_use_map_.erase(frame_iter);
}
void ChromeDataUseAscriber::DidStartMainFrameNavigation(
......@@ -158,36 +311,106 @@ void ChromeDataUseAscriber::ReadyToCommitMainFrameNavigation(
void* navigation_handle) {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
// TODO(kundaji): Move the DataUseRecorderEntry from pending navigation map
// to render frame map if |is_same_page_navigation| is true. Otherwise,
// merge it with the DataUseRecorderEntry in the render frame map.
DeletePendingNavigationEntry(global_request_id);
}
// Find the DataUseRecorderEntry the frame is associated with
auto frame_it = render_frame_data_use_map_.find(
RenderFrameHostID(render_process_id, render_frame_id));
void ChromeDataUseAscriber::DidRedirectMainFrameNavigation(
GURL gurl,
int render_process_id,
int render_frame_id,
void* navigation_handle) {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
}
void ChromeDataUseAscriber::DeletePendingNavigationEntry(
content::GlobalRequestID global_request_id) {
// Find the pending navigation entry.
auto navigation_iter =
pending_navigation_data_use_map_.find(global_request_id);
// Pending navigation entry will not be found if finish navigation
// raced the URLRequest.
if (navigation_iter != pending_navigation_data_use_map_.end()) {
auto entry = navigation_iter->second;
pending_navigation_data_use_map_.erase(navigation_iter);
// We might not find a navigation entry since the pending navigation may not
// have caused any HTTP or HTTPS URLRequests to be made.
if (navigation_iter == pending_navigation_data_use_map_.end()) {
// No pending navigation entry to worry about. However, the old frame entry
// must be removed from frame map, and possibly marked complete and deleted.
if (frame_it != render_frame_data_use_map_.end()) {
DataUseRecorderEntry old_frame_entry = frame_it->second;
render_frame_data_use_map_.erase(frame_it);
if (old_frame_entry->IsDataUseComplete()) {
OnDataUseCompleted(old_frame_entry);
data_use_recorders_.erase(old_frame_entry);
}
// Add a new recorder to the render frame map to replace the deleted one.
DataUseRecorderEntry entry = data_use_recorders_.emplace(
data_use_recorders_.end());
render_frame_data_use_map_.insert(std::make_pair(
RenderFrameHostID(render_process_id, render_frame_id), entry));
}
return;
}
DataUseRecorderEntry entry = navigation_iter->second;
pending_navigation_data_use_map_.erase(navigation_iter);
entry->set_main_frame_id(
RenderFrameHostID(render_process_id, render_frame_id));
// If the frame has already been deleted then mark this navigation as having
// completed its data use.
if (frame_it == render_frame_data_use_map_.end()) {
if (entry->IsDataUseComplete()) {
OnDataUseCompleted(entry);
data_use_recorders_.erase(entry);
}
return;
}
DataUseRecorderEntry old_frame_entry = frame_it->second;
if (is_same_page_navigation) {
old_frame_entry->MergeFrom(&(*entry));
for (auto request : entry->pending_url_requests()) {
request->RemoveUserData(DataUseRecorderEntryAsUserData::kUserDataKey);
request->SetUserData(DataUseRecorderEntryAsUserData::kUserDataKey,
new DataUseRecorderEntryAsUserData(old_frame_entry));
old_frame_entry->AddPendingURLRequest(request);
}
entry->RemoveAllPendingURLRequests();
data_use_recorders_.erase(entry);
} else {
// Navigation is not same page, so remove old entry from
// |render_frame_data_use_map_|, possibly marking it complete.
render_frame_data_use_map_.erase(frame_it);
if (old_frame_entry->IsDataUseComplete()) {
OnDataUseCompleted(old_frame_entry);
data_use_recorders_.erase(old_frame_entry);
}
DataUse& data_use = entry->data_use();
DCHECK(!data_use.url().is_valid() || data_use.url() == gurl)
<< "is valid: " << data_use.url().is_valid()
<< "; data_use.url(): " << data_use.url().spec()
<< "; gurl: " << gurl.spec();
if (!data_use.url().is_valid()) {
data_use.set_url(gurl);
}
render_frame_data_use_map_.insert(std::make_pair(
RenderFrameHostID(render_process_id, render_frame_id), entry));
}
}
void ChromeDataUseAscriber::OnDataUseCompleted(DataUseRecorderEntry entry) {
// TODO(ryansturm): Notify observers that data use is complete.
}
std::unique_ptr<URLRequestClassifier>
ChromeDataUseAscriber::CreateURLRequestClassifier() const {
return base::MakeUnique<ContentURLRequestClassifier>();
}
ChromeDataUseAscriber::DataUseRecorderEntry
ChromeDataUseAscriber::CreateNewDataUseRecorder(net::URLRequest* request) {
DataUseRecorderEntry entry = data_use_recorders_.emplace(
data_use_recorders_.end());
if (request) {
entry->AddPendingURLRequest(request);
request->SetUserData(DataUseRecorderEntryAsUserData::kUserDataKey,
new DataUseRecorderEntryAsUserData(entry));
}
return entry;
}
} // namespace data_use_measurement
......@@ -7,6 +7,8 @@
#include <list>
#include <memory>
#include <string>
#include <tuple>
#include <unordered_map>
#include <utility>
......@@ -14,6 +16,7 @@
#include "base/hash.h"
#include "base/macros.h"
#include "base/supports_user_data.h"
#include "chrome/browser/data_use_measurement/chrome_data_use_recorder.h"
#include "components/data_use_measurement/core/data_use_ascriber.h"
#include "content/public/browser/global_request_id.h"
#include "url/gurl.h"
......@@ -52,14 +55,10 @@ class ChromeDataUseAscriber : public DataUseAscriber {
~ChromeDataUseAscriber() override;
// DataUseAscriber implementation:
DataUseRecorder* GetDataUseRecorder(net::URLRequest* request) override;
// Called before a request is sent.
ChromeDataUseRecorder* GetDataUseRecorder(net::URLRequest* request,
bool can_create_new) override;
void OnBeforeUrlRequest(net::URLRequest* request) override;
// Called when a URLRequest is being destroyed.
void OnUrlRequestDestroyed(net::URLRequest* request) override;
std::unique_ptr<URLRequestClassifier> CreateURLRequestClassifier()
const override;
......@@ -91,21 +90,14 @@ class ChromeDataUseAscriber : public DataUseAscriber {
bool is_same_page_navigation,
void* navigation_handle);
// Called when a main frame navigation is redirected.
void DidRedirectMainFrameNavigation(GURL gurl,
int render_process_id,
int render_frame_id,
void* navigation_handle);
private:
// Use as a key in the render frame map. Corresponds to a unique
// RenderFrameHost.
typedef std::pair<int, int> RenderFrameHostID;
friend class ChromeDataUseAscriberTest;
// Entry in the |data_use_recorders_| list which owns all instances of
// DataUseRecorder.
typedef std::list<std::unique_ptr<data_use_measurement::DataUseRecorder>>::
iterator DataUseRecorderEntry;
typedef std::list<ChromeDataUseRecorder> DataUseRecorderList;
typedef DataUseRecorderList::iterator DataUseRecorderEntry;
struct GlobalRequestIDHash {
public:
......@@ -128,13 +120,22 @@ class ChromeDataUseAscriber : public DataUseAscriber {
DataUseRecorderEntry entry_;
};
void DeletePendingNavigationEntry(content::GlobalRequestID global_request_id);
DataUseRecorderEntry GetDataUseRecorderEntry(net::URLRequest* request,
bool can_create_new);
void OnDataUseCompleted(DataUseRecorderEntry entry);
DataUseRecorderEntry CreateNewDataUseRecorder(net::URLRequest* request);
bool IsRecorderInPendingNavigationMap(net::URLRequest* request);
bool IsRecorderInRenderFrameMap(net::URLRequest* request);
// Owner for all instances of DataUseRecorder. An instance is kept in this
// list if any entity (render frame hosts, URLRequests, pending navigations)
// that ascribe data use to the instance exists, and deleted when all
// ascribing entities go away.
std::list<std::unique_ptr<DataUseRecorder>> data_use_recorders_;
DataUseRecorderList data_use_recorders_;
// Map from RenderFrameHost to the DataUseRecorderEntry in
// |data_use_recorders_| that the frame ascribe data use to.
......
......@@ -115,13 +115,11 @@ void ChromeDataUseAscriberService::RenderFrameDeleted(
void ChromeDataUseAscriberService::DidStartNavigation(
content::NavigationHandle* navigation_handle) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
if (!navigation_handle->IsInMainFrame())
return;
if (!ascriber_)
return;
content::WebContents* web_contents = navigation_handle->GetWebContents();
content::BrowserThread::PostTask(
content::BrowserThread::IO, FROM_HERE,
......@@ -155,26 +153,6 @@ void ChromeDataUseAscriberService::ReadyToCommitNavigation(
navigation_handle));
}
void ChromeDataUseAscriberService::DidRedirectNavigation(
content::NavigationHandle* navigation_handle) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
if (!is_initialized_ || !navigation_handle->IsInMainFrame())
return;
if (!ascriber_)
return;
content::WebContents* web_contents = navigation_handle->GetWebContents();
content::BrowserThread::PostTask(
content::BrowserThread::IO, FROM_HERE,
base::Bind(&ChromeDataUseAscriber::DidRedirectMainFrameNavigation,
base::Unretained(ascriber_), navigation_handle->GetURL(),
web_contents->GetRenderProcessHost()->GetID(),
web_contents->GetMainFrame()->GetRoutingID(),
navigation_handle));
}
void ChromeDataUseAscriberService::SetDataUseAscriber(
ChromeDataUseAscriber* ascriber) {
DCHECK(!is_initialized_);
......
......@@ -53,11 +53,6 @@ class ChromeDataUseAscriberService : public KeyedService {
// methods cannot be called on the IO thread, so the pointer is cast to void*.
void ReadyToCommitNavigation(content::NavigationHandle* navigation_handle);
// Called when a navigation is redirected. Propagates main frame navigation
// redirect to the |ascriber_| on the IO thread. NavigationHandle methods
// cannot be called on the IO thread, so the pointer is cast to void*.
void DidRedirectNavigation(content::NavigationHandle* navigation_handle);
private:
friend class ChromeDataUseAscriberServiceTest;
......
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/data_use_measurement/chrome_data_use_ascriber.h"
#include <list>
#include <memory>
#include "base/message_loop/message_loop.h"
#include "base/run_loop.h"
#include "components/data_use_measurement/core/data_use_recorder.h"
#include "content/public/browser/resource_request_info.h"
#include "content/public/common/browser_side_navigation_policy.h"
#include "content/public/common/process_type.h"
#include "content/public/test/mock_resource_context.h"
#include "content/public/test/test_browser_thread_bundle.h"
#include "net/url_request/url_request_test_util.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace {
int kRenderProcessId = 1;
int kRenderFrameId = 2;
int kRequestId = 3;
}
namespace data_use_measurement {
class ChromeDataUseAscriberTest : public testing::Test {
protected:
ChromeDataUseAscriberTest()
: thread_bundle_(content::TestBrowserThreadBundle::IO_MAINLOOP),
resource_context_(new content::MockResourceContext(&context_)) {}
void SetUp() override {}
void TearDown() override { recorders().clear(); }
std::list<ChromeDataUseRecorder>& recorders() {
return ascriber_.data_use_recorders_;
}
net::TestURLRequestContext* context() { return &context_; }
content::MockResourceContext* resource_context() {
return resource_context_.get();
}
ChromeDataUseAscriber* ascriber() { return &ascriber_; }
std::unique_ptr<net::URLRequest> CreateNewRequest(std::string url,
bool is_main_frame,
int request_id,
int render_process_id,
int render_frame_id) {
std::unique_ptr<net::URLRequest> request =
context()->CreateRequest(GURL(url), net::IDLE, nullptr);
// TODO(kundaji): Allow request_id to be specified in AllocateForTesting.
content::ResourceRequestInfo::AllocateForTesting(
request.get(),