Skip to content

Commit

Permalink
fix: re-analyze youtube scripts api calls
Browse files Browse the repository at this point in the history
  • Loading branch information
@SichangHe
SichangHe committed Oct 10, 2024
1 parent 89ca1cd commit 579e41b
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 92 deletions.
2 changes: 1 addition & 1 deletion jsphere_vv8_log/src/aggregating.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ impl RecordAggregate {
this: method,
attr: None,
};
self.push_api_call(api_call, line)?;
self.push_api_call(api_call, line)?;
None
},

Expand Down
22 changes: 19 additions & 3 deletions jsphere_vv8_log/src/evcxr_examples.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,16 @@ fn main() {
}
}

let i_log = logs
.iter()
.enumerate()
.max_by_key(|(_, LogFile { records, .. })| records.len())
.map(|(i, _)| i)
.unwrap();

// Aggregate API calls.
let mut aggregate = RecordAggregate::default();
for entry in &logs[2].records {
for entry in &logs[i_log].records {
let (line, record) = entry.clone();
if let Err(err) = aggregate.add(line as u32, record) {
println!("{line}: {err}");
Expand All @@ -88,7 +95,16 @@ fn main() {
);
}

for (api_call, lines) in &aggregate.scripts[&23].api_calls {
println!("{} times: {api_call:?}", lines.lines.len());
for (id, script) in &aggregate.scripts {
for (api_call, lines) in &aggregate.scripts[&14].api_calls {
let n_may_interact = lines.n_may_interact();
if n_may_interact > 0 {
println!(
"{id}: {}/{} times: {api_call:?}",
n_may_interact,
lines.len()
);
}
}
}
}
192 changes: 104 additions & 88 deletions jsphere_vv8_log/youtube_scripts_api_calls_overview1009.md
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,21 @@
# Analysis of selected YouTube scripts API calls

Note: The `may_be_interaction` tags here are confusing.
They should have been per-line instead of per-API-type.

## Overview

```rust
let mut logs = read_logs("headless_browser/target/youtube.com/0").unwrap();
let i_log = logs
.iter()
.enumerate()
.max_by_key(|(_, LogFile { records, .. })| records.len())
.map(|(i, _)| i)
.unwrap();

// Aggregate API calls.
let mut aggregate = RecordAggregate::default();
for entry in &logs[2].records {
for entry in &logs[i_log].records {
let (line, record) = entry.clone();
if let Err(err) = aggregate.add(line, record) {
if let Err(err) = aggregate.add(line as u32, record) {
println!("{line}: {err}");
}
}
Expand All @@ -26,54 +31,53 @@ for (id, script) in &aggregate.scripts {
);
}

18 line#796 source~11kB used 18 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/www-tampering.vflset/www-tampering.js")
36 line#1090 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
14 line#1112 source~8466kB used 2362 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/desktop_polymer.vflset/desktop_polymer.js")
35 line#1078 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
10 line#372 source~0kB used 0 APIs Not Url("https://www.youtube.com/")
64 line#95189 source~0kB used 1 APIs Injected Eval { parent_script_id: 50 }
12 line#831 source~37kB used 46 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/spf.vflset/spf.js")
21 line#482 source~77kB used 459 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/webcomponents-sd.vflset/webcomponents-sd.js")
7 line#351 source~0kB used 2 APIs Not Url("https://www.youtube.com/")
17 line#782 source~5kB used 7 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/www-i18n-constants-en_US.vflset/www-i18n-constants.js")
26 line#986 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
48 line#89858 source~0kB used 0 APIs Not Url("https://www.youtube.com/")
56 line#94977 source~0kB used 4 APIs Injected Eval { parent_script_id: 50 }
30 line#1021 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
25 line#947 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
21 line#482 source~77kB used 459 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/webcomponents-sd.vflset/webcomponents-sd.js")
34 line#1066 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
43 line#89498 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
50 line#92474 source~11kB used 5 APIs Injected Empty
24 line#705 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
6 line#348 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
11 line#374 source~2kB used 15 APIs Not Url("https://www.youtube.com/")
15 line#413 source~50kB used 37 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/web-animations-next-lite.min.vflset/web-animations-next-lite.min.js")
22 line#595 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
19 line#717 source~9kB used 29 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/scheduler.vflset/scheduler.js")
33 line#1054 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
45 line#89653 source~0kB used 4 APIs Not Url("https://www.youtube.com/")
41 line#89054 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
65 line#260337 source~277kB used 214 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/www-searchbox.vflset/www-searchbox.js")
8 line#355 source~3kB used 6 APIs Not Url("https://www.youtube.com/")
34 line#1066 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
9 line#363 source~0kB used 5 APIs Not Url("https://www.youtube.com/")
32 line#1048 source~0kB used 2 APIs Not Url("https://www.youtube.com/")
20 line#467 source~2kB used 8 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/custom-elements-es5-adapter.vflset/custom-elements-es5-adapter.js")
13 line#951 source~14kB used 21 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/network.vflset/network.js")
27 line#998 source~0kB used 3 APIs Not Url("https://www.youtube.com/")
31 line#1036 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
6 line#348 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
23 line#607 source~445kB used 6 APIs Not Url("https://www.youtube.com/")
32 line#1048 source~0kB used 2 APIs Not Url("https://www.youtube.com/")
33 line#1054 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
36 line#1090 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
14 line#1112 source~8466kB used 2362 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/desktop_polymer.vflset/desktop_polymer.js")
61 line#95117 source~227kB used 135 APIs Injected Eval { parent_script_id: 50 }
65 line#260337 source~277kB used 214 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/www-searchbox.vflset/www-searchbox.js")
47 line#89775 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
46 line#89728 source~0kB used 14 APIs Not Url("https://www.youtube.com/")
50 line#92474 source~11kB used 5 APIs Injected Empty
58 line#94996 source~0kB used 4 APIs Injected Eval { parent_script_id: 50 }
64 line#95189 source~0kB used 1 APIs Interaction Eval { parent_script_id: 50 }
56 line#94977 source~0kB used 4 APIs Injected Eval { parent_script_id: 50 }
8 line#355 source~3kB used 6 APIs Not Url("https://www.youtube.com/")
41 line#89054 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
28 line#1009 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
42 line#89187 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
10 line#372 source~0kB used 0 APIs Not Url("https://www.youtube.com/")
55 line#94972 source~0kB used 0 APIs Injected Empty
45 line#89653 source~0kB used 4 APIs Not Url("https://www.youtube.com/")
49 line#89863 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
24 line#705 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
58 line#94996 source~0kB used 4 APIs Injected Eval { parent_script_id: 50 }
7 line#351 source~0kB used 2 APIs Not Url("https://www.youtube.com/")
22 line#595 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
()
13 line#951 source~14kB used 21 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/network.vflset/network.js")
27 line#998 source~0kB used 3 APIs Not Url("https://www.youtube.com/")
42 line#89187 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
37 line#1101 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
47 line#89775 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
16 line#591 source~5kB used 1 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/intersection-observer.min.vflset/intersection-observer.min.js")
43 line#89498 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
20 line#467 source~2kB used 8 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/custom-elements-es5-adapter.vflset/custom-elements-es5-adapter.js")
61 line#95117 source~227kB used 135 APIs Injected Eval { parent_script_id: 50 }
15 line#413 source~50kB used 37 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/web-animations-next-lite.min.vflset/web-animations-next-lite.min.js")
44 line#89581 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
12 line#831 source~37kB used 46 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/spf.vflset/spf.js")
16 line#591 source~5kB used 1 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/intersection-observer.min.vflset/intersection-observer.min.js")
48 line#89858 source~0kB used 0 APIs Not Url("https://www.youtube.com/")
23 line#607 source~445kB used 6 APIs Not Url("https://www.youtube.com/")
26 line#986 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
18 line#796 source~11kB used 18 APIs Not Url("https://www.youtube.com/s/desktop/72b8c307/jsbin/www-tampering.vflset/www-tampering.js")
30 line#1021 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
35 line#1078 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
37 line#1101 source~0kB used 1 APIs Not Url("https://www.youtube.com/")
```

## Selected scripts inspection
Expand All @@ -86,27 +90,31 @@ gathering information from `userAgent` and doing math:

```rust
for (api_call, lines) in &aggregate.scripts[&18].api_calls {
println!("{} times: {api_call:?}", lines.len());
println!(
"{}/{} times: {api_call:?}",
lines.n_may_interact(),
lines.len()
);
}

1 times: ApiCall { api_type: Set, this: "Window", attr: Some("ytbin"), may_be_interaction: false }
7 times: ApiCall { api_type: Get, this: "Window", attr: Some("navigator"), may_be_interaction: false }
1 times: ApiCall { api_type: Function, this: "Navigator", attr: Some("get userAgentData"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("CLOSURE_FLAGS"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("execScript"), may_be_interaction: false }
6 times: ApiCall { api_type: Get, this: "Navigator", attr: Some("userAgent"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("ytbin"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("Set"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Navigator", attr: Some("userAgentData"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("Object"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("WeakMap"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("Math"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("window"), may_be_interaction: false }
2 times: ApiCall { api_type: Get, this: "Window", attr: Some("Symbol"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("Map"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("String"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("Array"), may_be_interaction: false }
4 times: ApiCall { api_type: Get, this: "Window", attr: Some("yt"), may_be_interaction: false }
0/7 times: ApiCall { api_type: Get, this: "Window", attr: Some("navigator") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("Object") }
0/6 times: ApiCall { api_type: Get, this: "Navigator", attr: Some("userAgent") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("WeakMap") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("Math") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("Set") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("Array") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("execScript") }
0/1 times: ApiCall { api_type: Function, this: "Navigator", attr: Some("get userAgentData") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("window") }
0/4 times: ApiCall { api_type: Get, this: "Window", attr: Some("yt") }
0/2 times: ApiCall { api_type: Get, this: "Window", attr: Some("Symbol") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("CLOSURE_FLAGS") }
0/1 times: ApiCall { api_type: Get, this: "Navigator", attr: Some("userAgentData") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("String") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("ytbin") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("Map") }
0/1 times: ApiCall { api_type: Set, this: "Window", attr: Some("ytbin") }
```

---
Expand All @@ -121,29 +129,33 @@ These are mainly event-related APIs for **frontend processing** and APIs for
```rust
for (api_call, lines) in &aggregate.scripts[&14].api_calls {
if lines.len() > 2000 {
println!("{} times: {api_call:?}", lines.len());
println!(
"{}/{} times: {api_call:?}",
lines.n_may_interact(),
lines.len()
);
}
}

2480 times: ApiCall { api_type: Get, this: "Event", attr: Some("pageY"), may_be_interaction: false }
3031 times: ApiCall { api_type: Get, this: "c", attr: Some("polymerController"), may_be_interaction: false }
8817 times: ApiCall { api_type: Get, this: "Window", attr: Some("Reflect"), may_be_interaction: false }
45032 times: ApiCall { api_type: Get, this: "HTMLDivElement", attr: Some("tagName"), may_be_interaction: false }
7143 times: ApiCall { api_type: Function, this: "HTMLBodyElement", attr: Some("appendChild"), may_be_interaction: false }
2434 times: ApiCall { api_type: Get, this: "Event", attr: Some("type"), may_be_interaction: false }
2480 times: ApiCall { api_type: Get, this: "Event", attr: Some("target"), may_be_interaction: false }
3803 times: ApiCall { api_type: Function, this: "DocumentFragment", attr: Some("get children"), may_be_interaction: false }
7143 times: ApiCall { api_type: Get, this: "HTMLBodyElement", attr: Some("isConnected"), may_be_interaction: false }
2480 times: ApiCall { api_type: Get, this: "Event", attr: Some("pageX"), may_be_interaction: false }
21792 times: ApiCall { api_type: Function, this: "HTMLDivElement", attr: Some("querySelectorAll"), may_be_interaction: false }
6449 times: ApiCall { api_type: Get, this: "DocumentFragment", attr: Some("children"), may_be_interaction: false }
21780 times: ApiCall { api_type: Get, this: "HTMLDivElement", attr: Some("isConnected"), may_be_interaction: false }
2065 times: ApiCall { api_type: Get, this: "Event", attr: Some("keyCode"), may_be_interaction: false }
22409 times: ApiCall { api_type: Get, this: "HTMLDivElement", attr: Some("usePatchedLifecycles"), may_be_interaction: false }
10226 times: ApiCall { api_type: Get, this: "DocumentFragment", attr: Some("isConnected"), may_be_interaction: false }
10717 times: ApiCall { api_type: Function, this: "HTMLBodyElement", attr: Some("removeChild"), may_be_interaction: false }
2863 times: ApiCall { api_type: Get, this: "c", attr: Some("loggingStatus"), may_be_interaction: false }
10187 times: ApiCall { api_type: Function, this: "DocumentFragment", attr: Some("appendChild"), may_be_interaction: false }
2/2480 times: ApiCall { api_type: Get, this: "Event", attr: Some("target") }
146/6449 times: ApiCall { api_type: Get, this: "DocumentFragment", attr: Some("children") }
662/22409 times: ApiCall { api_type: Get, this: "HTMLDivElement", attr: Some("usePatchedLifecycles") }
3025/3031 times: ApiCall { api_type: Get, this: "c", attr: Some("polymerController") }
26/21780 times: ApiCall { api_type: Get, this: "HTMLDivElement", attr: Some("isConnected") }
0/8817 times: ApiCall { api_type: Get, this: "Window", attr: Some("Reflect") }
2/2480 times: ApiCall { api_type: Get, this: "Event", attr: Some("pageX") }
1/2434 times: ApiCall { api_type: Get, this: "Event", attr: Some("type") }
0/10717 times: ApiCall { api_type: Function, this: "HTMLBodyElement", attr: Some("removeChild") }
41/10226 times: ApiCall { api_type: Get, this: "DocumentFragment", attr: Some("isConnected") }
2/2480 times: ApiCall { api_type: Get, this: "Event", attr: Some("pageY") }
52/21792 times: ApiCall { api_type: Function, this: "HTMLDivElement", attr: Some("querySelectorAll") }
17/3803 times: ApiCall { api_type: Function, this: "DocumentFragment", attr: Some("get children") }
1/2065 times: ApiCall { api_type: Get, this: "Event", attr: Some("keyCode") }
4/7143 times: ApiCall { api_type: Function, this: "HTMLBodyElement", attr: Some("appendChild") }
2859/2863 times: ApiCall { api_type: Get, this: "c", attr: Some("loggingStatus") }
1411/45032 times: ApiCall { api_type: Get, this: "HTMLDivElement", attr: Some("tagName") }
2/10187 times: ApiCall { api_type: Function, this: "DocumentFragment", attr: Some("appendChild") }
4/7143 times: ApiCall { api_type: Get, this: "HTMLBodyElement", attr: Some("isConnected") }
```

---
Expand All @@ -155,13 +167,17 @@ the UI (e.g., `Downloading 1 video...`), thus kind of counts as

```rust
for (api_call, lines) in &aggregate.scripts[&23].api_calls {
println!("{} times: {api_call:?}", lines.len());
println!(
"{}/{} times: {api_call:?}",
lines.n_may_interact(),
lines.len()
);
}

1 times: ApiCall { api_type: Get, this: "Window", attr: Some("innerWidth"), may_be_interaction: false }
1 times: ApiCall { api_type: Set, this: "Window", attr: Some("ytplayer"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("yt"), may_be_interaction: false }
2 times: ApiCall { api_type: Get, this: "Window", attr: Some("ytcfg"), may_be_interaction: false }
1 times: ApiCall { api_type: Set, this: "Window", attr: Some("ytcfg"), may_be_interaction: false }
1 times: ApiCall { api_type: Get, this: "Window", attr: Some("innerHeight"), may_be_interaction: false }
0/1 times: ApiCall { api_type: Set, this: "Window", attr: Some("ytcfg") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("innerWidth") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("innerHeight") }
0/1 times: ApiCall { api_type: Set, this: "Window", attr: Some("ytplayer") }
0/2 times: ApiCall { api_type: Get, this: "Window", attr: Some("ytcfg") }
0/1 times: ApiCall { api_type: Get, this: "Window", attr: Some("yt") }
```

0 comments on commit 579e41b

Please sign in to comment.