From fc7bc737f342391a01bcc19b35c7f175f1fd1bf6 Mon Sep 17 00:00:00 2001 From: Uchio Kondo Date: Tue, 12 Oct 2021 17:16:58 +0900 Subject: [PATCH 1/6] Add reviews.rs --- src/lib.rs | 3 ++- src/reviews.rs | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 src/reviews.rs diff --git a/src/lib.rs b/src/lib.rs index abf1e4b..2c8b786 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ extern crate serde_urlencoded; +pub mod api_ext; pub mod comments; pub mod commits; pub mod events; @@ -7,10 +8,10 @@ pub mod issues; pub mod labels; pub mod pulls; pub mod releases; +pub mod reviews; pub mod users; pub mod users_detailed; pub mod workflows; -pub mod api_ext; use serde::Serialize; diff --git a/src/reviews.rs b/src/reviews.rs new file mode 100644 index 0000000..efd7495 --- /dev/null +++ b/src/reviews.rs @@ -0,0 +1,2 @@ +extern crate octocrab; +use serde::Serialize; From 977d6a1d3450110e0a92f36ed327165564ed7aad Mon Sep 17 00:00:00 2001 From: Uchio Kondo Date: Tue, 12 Oct 2021 17:20:01 +0900 Subject: [PATCH 2/6] Add --reviews opt --- src/main.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 73399cf..d4c4a67 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,8 +11,8 @@ extern crate octx; use octx::{ comments::CommentFetcher, commits::CommitFetcher, events::IssueEventFetcher, issues::IssueFetcher, labels::LabelFetcher, pulls::PullFileFetcher, releases::ReleaseFetcher, - users::UserFetcher, users_detailed::UserDetailedFetcher, workflows::WorkFlowFetcher, - workflows::RunFetcher, workflows::JobFetcher, + users::UserFetcher, users_detailed::UserDetailedFetcher, workflows::JobFetcher, + workflows::RunFetcher, workflows::WorkFlowFetcher, }; #[derive(StructOpt)] @@ -61,6 +61,9 @@ struct Command { /// Extract jobs #[structopt(long = "jobs")] target_jobs: bool, + /// Extract pull request reviews + #[structopt(long = "reviews")] + target_reviews: bool, /// Extract models created after N days ago. /// Only valid for --issues, --comments, --events --commits, --pull-request-files #[structopt(long = "days-ago")] @@ -165,6 +168,11 @@ async fn main() -> octocrab::Result<()> { info!("Target: jobs"); let runner = JobFetcher::new(owner, name, octocrab); runner.fetch(wtr, args.run_id).await?; + } else if args.target_reviews { + info!("Target: reviews"); + // let runner = JobFetcher::new(owner, name, octocrab); + // runner.fetch(wtr, args.run_id).await?; + unimplemented!("Hola") } else { error!("No target specified"); } From fdd442c2b6b20874fba01738d8e34232991c2d45 Mon Sep 17 00:00:00 2001 From: Uchio Kondo Date: Wed, 13 Oct 2021 16:31:01 +0900 Subject: [PATCH 3/6] Update models --- src/reviews.rs | 76 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/src/reviews.rs b/src/reviews.rs index efd7495..986187f 100644 --- a/src/reviews.rs +++ b/src/reviews.rs @@ -1,2 +1,76 @@ extern crate octocrab; -use serde::Serialize; +use octocrab::models::{pulls::ReviewState, User}; +use reqwest::Url; +use serde::{Deserialize, Serialize}; + +use crate::*; + +// Should support author_association: key. +// ref: https://docs.github.com/en/rest/reference/pulls#list-reviews-for-a-pull-request +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct Review { + pub id: u64, + pub node_id: String, + pub html_url: Url, + pub user: User, + #[serde(skip_serializing_if = "Option::is_none")] + pub body: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub commit_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub pull_request_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub submitted_at: Option>, + // omits links for our use case, for now + // #[serde(rename = "_links")] + // #[serde(skip_serializing_if = "Option::is_none")] + // pub links: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub author_association: Option, +} + +#[derive(Debug, Serialize)] +pub struct ReviewRec { + pub id: u64, + pub node_id: String, + pub html_url: Url, + pub user_id: i64, + pub body: Option, + pub commit_id: Option, + pub state: Option, + pub pull_request_url: Option, + pub submitted_at: Option>, + pub author_association: Option, + + pub pull_request_number: Option, + pub sdc_repository: String, +} + +impl RepositryAware for ReviewRec { + fn set_repository(&mut self, name: String) { + self.sdc_repository = name; + } +} + +impl From for ReviewRec { + fn from(from: Review) -> Self { + Self { + id: from.id, + node_id: from.node_id, + html_url: from.html_url, + user_id: from.user.id, + body: from.body, + commit_id: from.commit_id, + state: from.state, + pull_request_url: from.pull_request_url, + submitted_at: from.submitted_at, + author_association: from.author_association, + + pull_request_number: None, + sdc_repository: String::default(), + } + } +} From 04174912a689eec9bd10db26859e3c7db0da99b9 Mon Sep 17 00:00:00 2001 From: Uchio Kondo Date: Wed, 13 Oct 2021 16:55:14 +0900 Subject: [PATCH 4/6] Fetched pullz --- src/main.rs | 8 +++--- src/reviews.rs | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index d4c4a67..199d4ad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,8 +11,8 @@ extern crate octx; use octx::{ comments::CommentFetcher, commits::CommitFetcher, events::IssueEventFetcher, issues::IssueFetcher, labels::LabelFetcher, pulls::PullFileFetcher, releases::ReleaseFetcher, - users::UserFetcher, users_detailed::UserDetailedFetcher, workflows::JobFetcher, - workflows::RunFetcher, workflows::WorkFlowFetcher, + reviews::ReviewFetcher, users::UserFetcher, users_detailed::UserDetailedFetcher, + workflows::JobFetcher, workflows::RunFetcher, workflows::WorkFlowFetcher, }; #[derive(StructOpt)] @@ -170,8 +170,8 @@ async fn main() -> octocrab::Result<()> { runner.fetch(wtr, args.run_id).await?; } else if args.target_reviews { info!("Target: reviews"); - // let runner = JobFetcher::new(owner, name, octocrab); - // runner.fetch(wtr, args.run_id).await?; + let runner = ReviewFetcher::new(owner, name, since, octocrab); + runner.fetch(wtr).await?; unimplemented!("Hola") } else { error!("No target specified"); diff --git a/src/reviews.rs b/src/reviews.rs index 986187f..c015c78 100644 --- a/src/reviews.rs +++ b/src/reviews.rs @@ -1,10 +1,20 @@ extern crate octocrab; +use chrono::{DateTime, Utc}; use octocrab::models::{pulls::ReviewState, User}; use reqwest::Url; use serde::{Deserialize, Serialize}; use crate::*; +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct PullRequest { + pub id: u64, + + pub created_at: DateTime, + pub updated_at: Option>, +} + // Should support author_association: key. // ref: https://docs.github.com/en/rest/reference/pulls#list-reviews-for-a-pull-request #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -74,3 +84,62 @@ impl From for ReviewRec { } } } + +pub struct ReviewFetcher { + owner: String, + name: String, + since: Option>, + octocrab: octocrab::Octocrab, +} + +impl ReviewFetcher { + pub fn new( + owner: String, + name: String, + since: Option>, + octocrab: octocrab::Octocrab, + ) -> Self { + Self { + owner, + name, + since, + octocrab, + } + } +} + +impl ReviewFetcher { + pub async fn fetch(&self, mut wtr: csv::Writer) -> octocrab::Result<()> { + let param = Params::default(); + let route = format!( + "repos/{owner}/{repo}/pulls?{query}&state=all&sort=updated&direction=desc", + owner = &self.owner, + repo = &self.name, + query = param.to_query(), + ); + let mut next: Option = self.octocrab.absolute_url(route).ok(); + + let mut pull_ids: Vec = vec![]; + while let Some(mut page) = self.octocrab.get_page(&next).await? { + let pulls: Vec = page.take_items(); + let mut last_update: Option> = None; + for pull in pulls.into_iter() { + pull_ids.push(pull.id); + last_update = Some(pull.updated_at.unwrap_or_else(|| pull.created_at)); + } + + next = if let Some(since) = self.since { + if last_update.unwrap() < since { + None + } else { + page.next + } + } else { + page.next + }; + } + + eprintln!("{:?}", pull_ids); + Ok(()) + } +} From 23182fd0702615b86356771c8d71b040421172e5 Mon Sep 17 00:00:00 2001 From: Uchio Kondo Date: Wed, 13 Oct 2021 17:07:15 +0900 Subject: [PATCH 5/6] It works --- src/main.rs | 1 - src/reviews.rs | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/main.rs b/src/main.rs index 199d4ad..24636c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -172,7 +172,6 @@ async fn main() -> octocrab::Result<()> { info!("Target: reviews"); let runner = ReviewFetcher::new(owner, name, since, octocrab); runner.fetch(wtr).await?; - unimplemented!("Hola") } else { error!("No target specified"); } diff --git a/src/reviews.rs b/src/reviews.rs index c015c78..b797474 100644 --- a/src/reviews.rs +++ b/src/reviews.rs @@ -1,6 +1,6 @@ extern crate octocrab; use chrono::{DateTime, Utc}; -use octocrab::models::{pulls::ReviewState, User}; +use octocrab::models::User; use reqwest::Url; use serde::{Deserialize, Serialize}; @@ -9,7 +9,7 @@ use crate::*; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[non_exhaustive] pub struct PullRequest { - pub id: u64, + pub number: u64, pub created_at: DateTime, pub updated_at: Option>, @@ -29,7 +29,7 @@ pub struct Review { #[serde(skip_serializing_if = "Option::is_none")] pub commit_id: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub state: Option, + pub state: Option, #[serde(skip_serializing_if = "Option::is_none")] pub pull_request_url: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -50,12 +50,12 @@ pub struct ReviewRec { pub user_id: i64, pub body: Option, pub commit_id: Option, - pub state: Option, + pub state: Option, pub pull_request_url: Option, pub submitted_at: Option>, pub author_association: Option, - pub pull_request_number: Option, + pub pull_request_number: Option, pub sdc_repository: String, } @@ -119,12 +119,12 @@ impl ReviewFetcher { ); let mut next: Option = self.octocrab.absolute_url(route).ok(); - let mut pull_ids: Vec = vec![]; + let mut pull_nums: Vec = vec![]; while let Some(mut page) = self.octocrab.get_page(&next).await? { let pulls: Vec = page.take_items(); let mut last_update: Option> = None; for pull in pulls.into_iter() { - pull_ids.push(pull.id); + pull_nums.push(pull.number); last_update = Some(pull.updated_at.unwrap_or_else(|| pull.created_at)); } @@ -139,7 +139,28 @@ impl ReviewFetcher { }; } - eprintln!("{:?}", pull_ids); + for number in pull_nums.into_iter() { + let param = Params::default(); + let route = format!( + "repos/{owner}/{repo}/pulls/{pull_number}/reviews?{query}", + owner = &self.owner, + repo = &self.name, + pull_number = number, + query = param.to_query(), + ); + let mut next: Option = self.octocrab.absolute_url(route).ok(); + while let Some(mut page) = self.octocrab.get_page(&next).await? { + let reviews: Vec = page.take_items(); + for review in reviews.into_iter() { + let mut review: ReviewRec = review.into(); + review.sdc_repository = format!("{}/{}", self.owner, self.name); + review.pull_request_number = Some(number); + + wtr.serialize(review).expect("Serialize failed"); + } + next = page.next; + } + } Ok(()) } } From 9049a4815a834148ceeb26c3e4601e808f2722da Mon Sep 17 00:00:00 2001 From: Uchio Kondo Date: Wed, 13 Oct 2021 19:53:56 +0900 Subject: [PATCH 6/6] Update version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e6611b5..40365dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "octx" -version = "0.5.0" +version = "0.5.1" authors = ["Uchio Kondo "] repository = "https://github.com/pepabo/octx" keywords = ["github", "cli", "etl"]