Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix remote execution regression #72

Merged
merged 4 commits into from
Aug 26, 2022
Merged

fix remote execution regression #72

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5d5f78d
github action: fix event payload type of repository dispatch, because…
umegaya Aug 26, 2022
8820525
fix repository dispatch problems. now ci::filter_workflow directly re…
umegaya Aug 26, 2022
285ac27
separate run/boot handler
umegaya Aug 26, 2022
ebfc85c
apply revision/silent/verbosity correctly
umegaya Aug 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions .github/workflows/deplo-main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,8 @@ on:

env:
DEPLO_GHACTION_CI_ID: ${{ github.run_id }}-${{ github.run_attempt }}
DEPLO_GHACTION_PR_URL: ${{ github.event.pull_request.url }}
DEPLO_GHACTION_EVENT_DATA: ${{ toJson(github) }}
DEPLO_OVERWRITE_COMMIT: ${{ github.event.client_payload.commit }}
DEPLO_OVERWRITE_RELEASE_TARGET: ${{ github.event.client_payload.release_target }}
DEPLO_OVERWRITE_VERBOSITY: ${{ github.event.client_payload.verbosity }}
DEPLO_OVERWRITE_WORKFLOW: ${{ github.event.client_payload.workflow }}
DEPLO_OVERWRITE_VERBOSITY: ${{ github.event.client_payload.exec.verbosity }}
SUNTOMI_VCS_ACCOUNT: ${{ secrets.SUNTOMI_VCS_ACCOUNT }}
SUNTOMI_VCS_ACCOUNT_EMAIL: ${{ secrets.SUNTOMI_VCS_ACCOUNT_EMAIL }}
SUNTOMI_VCS_ACCOUNT_KEY: ${{ secrets.SUNTOMI_VCS_ACCOUNT_KEY }}
Expand Down
4 changes: 2 additions & 2 deletions .vscode/launch.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,12 @@
]
},
{
"name": "Start",
"name": "Boot",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/target/debug/cli",
"args": [
"-v=3", "start", "-r=nightly", "-w=integrate"
"-v=3", "boot", "-p=payload.json"
]
},
{
Expand Down
4 changes: 2 additions & 2 deletions Deplo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,8 +188,8 @@ docker push ghcr.io/suntomi/deplo:${DEPLO_RELEASE_VERSION}
# deplo provides unified command for controlling vcs.
# `deplo vcs release` and `deplo vcs release-assets` are used for creating release of vcs like github/gitlab
# with unified interface.
deplo -v=3 vcs release ${DEPLO_RELEASE_TAG} -o name=${DEPLO_RELEASE_NAME}
deplo -v=3 vcs release-assets ${DEPLO_RELEASE_TAG} tools/docker/bin/cli --replace -o name=deplo-Linux
deplo vcs release ${DEPLO_RELEASE_TAG} -o name=${DEPLO_RELEASE_NAME}
deplo vcs release-assets ${DEPLO_RELEASE_TAG} tools/docker/bin/cli --replace -o name=deplo-Linux

# deplo provides dyanamically generated job output. you can use value for 'foo' and 'fuga' in jobs that depends on the job.
deplo job set-output foo bar
Expand Down
2 changes: 1 addition & 1 deletion cli/src/command/run.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ impl<S: shell::Shell, A: args::Args> command::Command<A> for Run<S> {
self.config.prepare_workflow()?;
let workflow = config::runtime::Workflow::new(args, &self.config, true)?;
let config = self.config.borrow();
config.jobs.boot(&config, &workflow, &self.shell)?;
config.jobs.run(&config, &workflow, &self.shell)?;
return Ok(());
}
}
6 changes: 1 addition & 5 deletions core/res/ci/ghaction/main.yml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,8 @@ on:

env:
DEPLO_GHACTION_CI_ID: ${{{{ github.run_id }}}}-${{{{ github.run_attempt }}}}
DEPLO_GHACTION_PR_URL: ${{{{ github.event.pull_request.url }}}}
DEPLO_GHACTION_EVENT_DATA: ${{{{ toJson(github) }}}}
DEPLO_OVERWRITE_COMMIT: ${{{{ github.event.client_payload.commit }}}}
DEPLO_OVERWRITE_RELEASE_TARGET: ${{{{ github.event.client_payload.release_target }}}}
DEPLO_OVERWRITE_VERBOSITY: ${{{{ github.event.client_payload.verbosity }}}}
DEPLO_OVERWRITE_WORKFLOW: ${{{{ github.event.client_payload.workflow }}}}
DEPLO_OVERWRITE_VERBOSITY: ${{{{ github.event.client_payload.exec.verbosity }}}}
{secrets:>2}
jobs:
deplo-main:
Expand Down
2 changes: 1 addition & 1 deletion core/src/ci.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ pub trait CI {
fn process_env(&self) -> Result<HashMap<&str, String>, Box<dyn Error>>;
fn filter_workflows(
&self, trigger: Option<WorkflowTrigger>
) -> Result<Vec<(String, HashMap<String, config::AnyValue>)>, Box<dyn Error>>;
) -> Result<Vec<config::runtime::Workflow>, Box<dyn Error>>;
fn set_job_output(&self, job_name: &str, kind: OutputKind, outputs: HashMap<&str, &str>) -> Result<(), Box<dyn Error>>;
fn job_output(&self, job_name: &str, kind: OutputKind, key: &str) -> Result<Option<String>, Box<dyn Error>>;
}
Expand Down
2 changes: 1 addition & 1 deletion core/src/ci/circleci.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ impl<'a, S: shell::Shell> ci::CI for CircleCI<S> {
}
fn filter_workflows(
&self, _trigger: Option<ci::WorkflowTrigger>
) -> Result<Vec<(String, HashMap<String, config::AnyValue>)>, Box<dyn Error>> {
) -> Result<Vec<config::runtime::Workflow>, Box<dyn Error>> {
log::warn!("TODO: implement filter_workflows for circleci");
Ok(vec![])
}
Expand Down
47 changes: 26 additions & 21 deletions core/src/ci/ghaction.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use chrono::{Utc, Duration};
use log;
use maplit::hashmap;
use serde::{Deserialize, Serialize};
use serde_json::{Value as JsonValue};

use crate::config;
use crate::ci;
Expand All @@ -34,7 +35,7 @@ enum EventPayload {
// to avoid wrongly being matched as 'Repository' variant.
RepositoryDispatch {
action: String,
client_payload: config::AnyValue
client_payload: JsonValue
},
Repository {
action: Option<String>
Expand Down Expand Up @@ -569,7 +570,7 @@ impl<S: shell::Shell> ci::CI for GhAction<S> {
}
fn filter_workflows(
&self, trigger: Option<ci::WorkflowTrigger>
) -> Result<Vec<(String, HashMap<String, config::AnyValue>)>, Box<dyn Error>> {
) -> Result<Vec<config::runtime::Workflow>, Box<dyn Error>> {
let resolved_trigger = match trigger {
Some(t) => t,
// on github action, full event payload is stored env var 'DEPLO_GHACTION_EVENT_DATA'
Expand Down Expand Up @@ -601,28 +602,30 @@ impl<S: shell::Shell> ci::CI for GhAction<S> {
_ => {}
},
// repository_dispatch has a few possibility.
// config::DEPLO_REMOTE_JOB_EVENT_TYPE => should contain workflow_name in client_payload
// config::DEPLO_REMOTE_JOB_EVENT_TYPE => should contain workflow name in client_payload["name"]
// config::DEPLO_MODULE_EVENT_TYPE => Module workflow invocation
// others => Repository workflow invocation
"repository_dispatch" => if let EventPayload::RepositoryDispatch{
action, client_payload
} = &workflow_event.event {
if action == config::DEPLO_REMOTE_JOB_EVENT_TYPE {
match client_payload.index("workflow_name") {
Some(n) => match n.as_str() {
Some(s) => matched_names.push(s.to_string()),
None => panic!(
"{}: event payload invalid {}",
config::DEPLO_REMOTE_JOB_EVENT_TYPE, client_payload
)
match &client_payload["name"] {
JsonValue::String(s) => {
let workflow_name = s.to_string();
if workflow_name == name {
return Ok(vec![config::runtime::Workflow::with_payload(
&serde_json::to_string(client_payload)?
)?]);
}
},
None => panic!(
_ => panic!(
"{}: event payload invalid {}",
config::DEPLO_REMOTE_JOB_EVENT_TYPE, client_payload
)
}
} else if action == config::DEPLO_MODULE_EVENT_TYPE {
if let config::workflow::Workflow::Module(..) = v {
log::warn!("TODO: should check current workflow is matched for the module?");
matched_names.push(name);
}
} else if let config::workflow::Workflow::Repository{..} = v {
Expand All @@ -644,18 +647,20 @@ impl<S: shell::Shell> ci::CI for GhAction<S> {
match config.workflows.get(&name).expect(&format!("workflow {} not found", name)) {
config::workflow::Workflow::Deploy|config::workflow::Workflow::Integrate => {
let target = vcs.release_target();
matches.push((name, if target.is_none() { hashmap!{} } else { hashmap!{
"release_target".to_string() => config::AnyValue::new(&target.unwrap())
}}))
matches.push(config::runtime::Workflow::with_context(
name, if target.is_none() { hashmap!{} } else { hashmap!{
"release_target".to_string() => config::AnyValue::new(&target.unwrap())
}}
))
},
config::workflow::Workflow::Cron{schedules} => {
if let EventPayload::Schedule{ref schedule} = workflow_event.event {
match schedules.iter().find_map(|(k, v)| {
if v.resolve().as_str() == schedule.as_str() { Some(k) } else { None }
}) {
Some(schedule_name) => matches.push((name, hashmap!{
"schedule".to_string() => config::AnyValue::new(schedule_name)
})),
Some(schedule_name) => matches.push(config::runtime::Workflow::with_context(
name, hashmap!{ "schedule".to_string() => config::AnyValue::new(schedule_name) }
)),
None => {}
}
} else {
Expand All @@ -674,16 +679,16 @@ impl<S: shell::Shell> ci::CI for GhAction<S> {
match events.iter().find_map(|(k, vs)| {
if vs.iter().find(|t| t == &key).is_some() { Some(k) } else { None }
}) {
Some(event_name) => matches.push((name, hashmap!{
"event".to_string() => config::AnyValue::new(event_name)
})),
Some(event_name) => matches.push(config::runtime::Workflow::with_context(
name, hashmap!{ "event".to_string() => config::AnyValue::new(event_name) }
)),
None => {}
}
} else {
panic!("event payload type does not match {}", workflow_event.event);
}
},
config::workflow::Workflow::Module(c) => {
config::workflow::Workflow::Module(_c) => {
panic!("not implemented yet")
}
}
Expand Down
15 changes: 13 additions & 2 deletions core/src/config/job.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -790,14 +790,25 @@ impl Jobs {
}
log::info!("remote job {} id={} finished", job_name, job_id);
Ok(())
}
}
pub fn run<S>(
&self, config: &config::Config, runtime_workflow_config: &config::runtime::Workflow, shell: &S
) -> Result<(), Box<dyn Error>> where S: shell::Shell {
let name = &runtime_workflow_config.job.as_ref().expect("should have job setting").name;
let job = config.jobs.find(name).expect(&format!("job '{}' does not exist", name));
match job.run(shell, config, runtime_workflow_config)? {
Some(job_id) => self.wait_job(&job_id, name, config, runtime_workflow_config)?,
None => {}
};
Ok(())
}
pub fn boot<S>(
&self, config: &config::Config, runtime_workflow_config: &config::runtime::Workflow, shell: &S
) -> Result<(), Box<dyn Error>> where S: shell::Shell {
let modules = &config.modules;
let (_, ci) = modules.ci_by_env();
for (name, job) in self.filter_as_map(config, runtime_workflow_config) {
if config::Config::is_running_on_ci() && runtime_workflow_config.job.is_none() {
if config::Config::is_running_on_ci() {
ci.schedule_job(name)?;
} else {
match job.run(shell, config, runtime_workflow_config)? {
Expand Down
22 changes: 16 additions & 6 deletions core/src/config/job/runner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,17 @@ impl<'a> Runner<'a> {
}
fn adjust_commit_hash(&self, commit: &Option<&str>) -> Result<(), Box<dyn Error>> {
let config = self.config;
if !config::Config::is_running_on_ci() {
if let Some(ref c) = commit {
if let Some(ref c) = commit {
let vcs = config.modules.vcs();
if config::Config::is_running_on_ci() {
let target = vcs.commit_hash(Some(c))?;
let current = vcs.commit_hash(None)?;
if target != current {
panic!("on CI, HEAD should already set to '{}' but '{}'", target, current);
}
} else {
log::debug!("change commit hash to {}", c);
config.modules.vcs().checkout(c, Some(config::DEPLO_VCS_TEMPORARY_WORKSPACE_NAME))?;
vcs.checkout(c, Some(config::DEPLO_VCS_TEMPORARY_WORKSPACE_NAME))?;
}
}
Ok(())
Expand Down Expand Up @@ -116,7 +123,13 @@ impl<'a> Runner<'a> {
let config = self.config;
let job = self.job;
let exec = &runtime_workflow_config.exec;
// apply exec settings to current workspace.
// verbosity is set via envvar DEPLO_OVERWRITE_VERBOSITY
// revision
self.adjust_commit_hash(&exec.revision.as_ref().map(|v| v.as_str()))?;
defer!{self.recover_branch().unwrap();};
let command = runtime_workflow_config.command();
// silent
let shell_settings = &mut match command {
job::Command::Shell => shell::interactive(),
_ => if exec.silent {
Expand All @@ -126,9 +139,6 @@ impl<'a> Runner<'a> {
}
};
let (steps, main_command) = self.create_steps(&command);
// if current commit is modified, rollback after all operation is done.
self.adjust_commit_hash(&exec.revision.as_ref().map(|v| v.as_str()))?;
defer!{self.recover_branch().unwrap();};
if exec.remote {
log::debug!(
"force running job {} on remote with steps {} at {}",
Expand Down
Loading