[{"_1":2,"_169":-5,"_170":-5},"loaderData",{"_3":4,"_13":14,"_20":21,"_148":149},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":11,"_17":11,"_18":19},"username","isBillingDecouplingSeatCount","isSearchBoxExperimentEnabled","isAskGordonEnabled",true,"routes/_layout.r.$namespace.$reponame",{"_22":23,"_24":-5,"_25":11,"_26":11,"_27":28,"_29":30,"_31":-5,"_32":11,"_33":34,"_83":84,"_103":104,"_141":-5,"_142":11,"_143":11,"_144":11,"_145":-5,"_146":147},"canonicalUrl","https://hub.docker.com/r/rustlang/crater/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","rustlang","reponame","crater","cloudBuildSource","isAutobuildsAvailable","repo",{"_35":36,"_37":38,"_40":41,"_42":43,"_44":11,"_45":46,"_47":48,"_49":50,"_52":30,"_27":28,"_53":54,"_55":39,"_56":57,"_58":59,"_60":61,"_62":-5,"_63":57,"_64":65,"_66":11,"_67":28,"_68":69,"_74":19,"_75":76,"_80":28,"_81":82},"categories",[],"content_types",[39],"image","date_registered","2017-10-16T17:15:40.590299Z","description","Crater testing image","is_private","last_updated","2017-12-19T23:53:10.568586Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[51],"application/vnd.docker.container.image.v1+json","name","pull_count",224,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","# Crater\n\n[![Build Status](https://travis-ci.org/rust-lang-nursery/crater.svg?branch=master)](https://travis-ci.org/rust-lang-nursery/crater)\n\nCrater is a laboratory for running experiments across a large body\nof Rust source code. Its primary purpose is to detect regressions in\nthe Rust compiler, and it does this by building large numbers of\ncrates, running their test suites, and comparing the results between\ntwo versions of Rust.\n\nIt can operate completely locally, with only a dependency on docker,\nor it can run distributed on AWS. It should work on Windows.\n\nSome of the goals of Crater:\n\n- Discover Rust codebases from crates.io and GitHub\n- Download all Rust code to a local disk\n- Build and manage custom Rust toolchains\n- Run `cargo build` and `cargo test` over all codebases\n- Cache dependencies to avoid unnecessary rebuilds\n- Lockfiles shared between runs\n- Dependencies fetched ahead of time\n- Building and testing is `--frozen` - no dependency updates or network access\n- Run arbitrary tests over all codebases\n- Resume partial test runs\n- Generate summary HTML and text reports\n- Run on Linux and Windows\n- Isolate tests into docker containers on Linux and Windows\n- Test against Linux-based cross targets under docker\n- Hosted, distributed testing on AWS\n\nCrater is a successor to https://github.com/brson/taskcluster-crater.\nIt was subsequently named cargobomb before resuming the Crater name, so for now the code still refers to cargobomb in many places (Being addressed in #134).\n\n__Warning: do not run Crater in an unsandboxed environment.__\n___Crater executes malicious code that will destroy what you love.___\n\n\n## Basic local use\n\nThese commands will run Crater, in local configuration, on the demo\ncrate set. This is safe to run unsanboxed because the set of crates\ntested is limited to the 'demo' set. This requires the user have\naccess to the docker daemon.\n\nToday Crater expects to be run out of its source directory, and all\nof its output is into the `./work` directory, where it maintains its\nown rustup installation, crate mirrors, etc.\n\n```\ncargo run -- prepare-local --docker-env mini\ncargo run -- define-ex --crate-select=demo stable beta\ncargo run -- prepare-ex\ncargo run -- run\ncargo run -- gen-report work/ex/default/\n```\n\nThis will output a report to `./work/ex/default/index.html`.\n\nDelete things with\n```\ncargo run -- delete-all-target-dirs\ncargo run -- delete-all-results\ncargo run -- delete-ex\n```\nEach command except `prepare-local` optionally takes an `--ex` argument\nto identify the experiment being referred to. If not supplied, this\ndefaults to `default`. Here's what each of the steps does:\n\n* `prepare-local` - sets up the stable toolchain for internal use,\n builds the docker container, builds lists of crates. This needs to\n be rerun periodically, but not between every experiment.\n\n* `define-ex` - defines a new experiment\n performing a build-test experiment on the 'demo' set of crates.\n\n* `prepare-ex` - fetches repos from github and captures their commit\n shas, downloads all crates, hacks up Cargo.toml files, captures\n lockfiles, fetches all dependencies, and prepares toolchains.\n\n* `run` - runs tests on crates in the experiment, against both\n toolchains\n\n* `gen-report` - summarize the experiment results to\n work/ex/default/index.html\n\n* `delete-all-target-dirs`/`delete-all-results`/`delete-ex` - clean up\n everything relating to this experiment\n\n### Custom toolchains\n\nToolchains for rust PRs that have been built by by asking bors to try a PR can\nbe specified using `try#`. You will probably want to specify\nthe comparison commit as `master#`.\n\n## Operational workflow for production Crater\n\n### Getting access\n\nThere are three 'official' Crater machines:\n\n - cargobomb-test (54.177.234.51) - 1 core, 4GB RAM, for experimenting\n - cargobomb-try (54.241.86.211) - 8 core, 30GB RAM, for doing PR runs\n - cargobomb-prod (54.177.126.219) - 8 core, 30GB RAM, for doing beta runs (but can do PR runs if free)\n\nThese can only be accessed via the bastion - you `ssh` to the bastion,\nthen `ssh` to the Crater machine. The bastion has restricted access\nand you will need a static IP address (if you have a long-running server\nin the cloud, that's usually fine) and a public SSH key (you should add\nthe key to github and then link to https://github.com/yourusername.keys,\nonce you have access to the bastion you can manage your own keys).\n\nWith these two pieces of information in hand, ask acrichto to\nadd you to the bastion and all three machines and they'll let you know\nthe bastion IP. You can now either edit your `~/.ssh/config` on your\nstatic IP machine to contain\n\n```\nHost rust-bastion\n # Bastion IP below\n HostName 0.0.0.0\n User bastionusername\nHost cargobomb-test\n HostName 54.177.234.51\n ProxyCommand ssh -q rust-bastion nc -q0 %h 22\n User ec2-user\n# [...and so on for cargobomb-try and cargobomb-prod...]\n```\n\nwhich will let you do `ssh cargobomb-test` etc from your static IP\nmachine. If you have a recent OpenSSH, you can use `ProxyJump` instead.\n\n### General Crater server notes\n\nThe Crater servers use a terminal multiplexer (a way to keep multiple\nterminals running on a server). Enter the multiplexer by logging onto a\nserver and running `byobu`. You'll notice a bit of text along the\nbottom saying something like \"0:master 1:tc1 2:tc2\" - these are\nthe 'windows' in the terminal multiplexer. The one highlighted and with a\n`*` next to it is the current window. Sending commands to the multiplexer\nis achieved by pressing Ctrl+Z, and then another key.\n\nSome useful operations:\n - Ctrl+Z d - detach from the multiplexer (or you can just close your\n terminal)\n - Ctrl+Z 0 - switch to window 0 (or any other number)\n - Ctrl+Z PageUp - scroll upwards on the terminal. This will enter a\n sort of 'scrolling mode', so you can use PageUp and PageDown freely\n (to the limit of terminal scrollback). To return to normal terminal\n mode, hit Ctrl+C - be sure to only press it once, or you risk\n returning to normal mode and then killing the process running in the\n current terminal!\n - Ctrl+Z c - create a new window, useful if you accidentally closed one\n - Ctrl+Z , - rename a window, useful after recreating an accidentally\n closed window (hit enter to accept new name)\n\n### Crater triage process\n\nOn your day for Crater triage, open the\n[sheet](https://docs.google.com/spreadsheets/d/1VPi_7ErvvX76fa3VqvQ3YnQmDk3bS7fYnkzvApIWkKo/edit#gid=0).\nClick the top left cell and make sure every PR on that list has an\nentry on the sheet and make sure every row on the sheet without\n'Complete' or 'Failed' is listed on the GitHub search. You may need\nto update PR tags or add rows to the sheet as appropriate.\n\nNext, you should follow the steps below for eachrequested run on\nthe sheet that does not have a status of 'Complete' or 'Failed'.\n\n - Pending\n - Is try or prod available? (prioritise beta runs to go on prod, no\n matter how far down the pending list they are) If not, go to next run.\n - Log onto appropriate box and connect to multiplexer.\n - Double check each multiplexer window to make sure nothing is\n running.\n - Switch to the `master` multiplexer window.\n - Run `docker ps` to make sure no containers are running.\n - Run `df -h /home/ec2-user/cargobomb/work`, disk usage should be\n <250GB of the 1TB disk (a full run may consume 600GB)\n - If disk usage is greater, there are probably target directories\n left over from a previous run. Run `du -sh work/local/target-dirs/*`,\n find the culprit (likely a directory with >100GB).\n - The directory name is the name of an experiment, e.g. MY_EX, so run\n `cargo run --release -- delete-all-target-dirs --ex MY_EX`.\n - Run `docker ps -aq | xargs --no-run-if-empty docker rm` to clean up all terminated\n Docker containers.\n - Run `git stash && git pull && git stash pop` to get the latest Crater changes.\n If this fails, it means there were local changes that conflict with upstream\n changes. Ping aidanhs and tomprince on IRC.\n - Run `cargo run --release -- prepare-local`. This may take between 5s and 5min, depending\n on what needs doing.\n - Log `EX_NAME`, `EX_START` and `EX_END` in the spreadsheet, where:\n - If doing a run for PR 12345, `EX_NAME` is `pr-12345`, `EX_END` is\n `try#deadbeef2...` (`deadbeef2` is in the bors comment \"Trying commit\n `abcdef` with merge `deadbeef2`\" - click through and copy from the URL to get the full\n commitish) and `EX_START` is `master#deadbeef1...` (`deadbeef1` is on the page you clicked\n through to get `deadbeef2...`, just below the commit message, the left hand commit of\n \"2 parents `deadbeef1` and `bcdef1`\" - click through and copy from the URL to get the\n full commitish, make sure the commit is an auto merge from bors). Just to emphasise,\n the second commitish you copied goes in `EX_START`.\n - If doing a beta run, `EX_NAME` is `stable-STABLE_VERSION-beta-BETA_VERSION`, `EX_START` is\n `LAST_STABLE` and `EX_END` is `BETA_DATE`. `STABLE_VERSION` is the version number from\n `curl -sSL static.rust-lang.org/dist/channel-rust-stable.toml | grep -A1 -F '[pkg.rust]'`,\n `BETA_VERSION` is the version number from\n `curl -sSL static.rust-lang.org/dist/channel-rust-beta.toml | grep -A1 -F '[pkg.rust]'`\n and `BETA_DATE` is the date from\n `curl -sSL static.rust-lang.org/dist/channel-rust-beta.toml | grep '^date ='` (it is *not*\n necessarily the same date as retrieved in the `BETA_VERSION` command).\n - Run `cargo run --release -- define-ex --crate-select=full --ex EX_NAME EX_START EX_END`.\n This will complete in a few seconds.\n - Run `cargo run --release -- prepare-ex --ex EX_NAME`.\n - Change status to 'Preparing'.\n - Update either the PR or the person requesting the run to let them know the run has started.\n - Go to next run.\n - Preparing\n - Log onto appropriate box and connect to multiplexer.\n - Switch to the `master` multiplexer window.\n - If preparation is ongoing, go to next run.\n - If preparation failed, fix it. Known errors:\n - \"missing sha for ...\" - remove the referenced repository from `gh-apps.txt`\n and `gh-candidates.txt` (may be present in one or both). Make the same\n change locally and make a PR against Crater. Use\n `cargo run --release -- delete-all-target-dirs --ex EX_NAME` and\n `cargo run --release -- delete-ex --ex EX_NAME`, then jump to start of 'Pending'.\n - Switch to the `tc1` multiplexer window.\n - Run `cargo run --release -- run-tc --ex EX_NAME EX_START`.\n - Switch to the `tc2` multiplexer window.\n - Run `cargo run --release -- run-tc --ex EX_NAME EX_END`.\n - Go to next run.\n - Running\n - Log onto appropriate box and connect to multiplexer.\n - Switch to the `master` multiplexer window.\n - Run `docker ps`. If any container has been running for more than 30min (may\n need to follow these steps more than once):\n - Take solace in us someday fixing this for good with docker limits.\n TODO: actually fix. Seems to only be a problem on prod with pleingres,\n our existing limits should catch it.\n - Run `docker top CONTAINER_ID`.\n - If there's no mention of pleingres, raise an issue with the output of\n the previous `docker top` command.\n - The process at the bottom of the list is the lowest in the process tree,\n and should have a value in the `TIME` column of >30min. Find the value in\n the `PID` column and run `kill PID`.\n - Wait a few seconds, then check the container has now exited.\n - If the run is ongoing in either the `tc1` or `tc2` multiplexer\n windows, go to next run.\n - Switch to the `master` multiplexer window.\n - Run `du -sh work/ex/EX_NAME`, output should be <2GB. If not:\n - Run `find work/ex/EX_NAME -type f -size +100M | xargs du -sh`,\n there will likely only be a couple of files listed and they\n should be in the `res` directory (TODO: blacklist pleingres\n as the main culprit here once it's possible, and update\n these instructions to suggest adding things to the blacklist).\n - For each file found, run `truncate --size='<100M' FILE`.\n - Check `du -sh work/ex/EX_NAME` is now an appropriate size.\n - Run `cargo run --release -- publish-report --ex EX_NAME s3://cargobomb-reports/EX_NAME`.\n - Change status to 'Uploading'.\n - (optional but much appreciated: come back to this run in 30mins\n as the upload will be complete)\n - Go to next run.\n - Uploading\n - Switch to the `master` multiplexer window.\n - If the upload is ongoing, go to the next run.\n - If the upload failed, fix it. Known errors:\n - `InternalError...` - probably an s3 failure, try running\n upload again.\n - Run `cargo run --release -- delete-all-target-dirs --ex EX_NAME`. This will take ~2min.\n - Change status to 'Complete' and add the results link,\n `http://cargobomb-reports.s3.amazonaws.com/EX_NAME/index.html`.\n - Update either the PR or the person requesting the beta run. Template is:\n ```\n Crater results: . 'Blacklisted' crates (spurious failures etc) can be found\n [here](https://github.com/rust-lang-nursery/crater/blob/master/blacklist.md).\n If you see any spurious failures not on the list, please make a PR against that file.\n ```\n - Give yourself a pat on the back! Good job!\n - Go to next run.\n\n(The runs can be stopped and restarted at any time. - really? How? asks aidanhs)\n\nIf a beta run has completed, regressions need reporting (PR runs are left to the\npeople involved in the PR). To report regressions you'll need to\nnavigate to the results page, wait for a bit (<30s) for the results to load (the\nbuttons will be populated with numbers) and then click 'regressed'. The triage\nprocess (e.g. checking the cause of a regression) is 'crowd-sourced', we just\nreport the issues (for now).\n\nYou can follow whatever process you like for working through regressions,\nbut a suggestion workflow is described below, per regression:\n\n - Open the regression log (i.e. 'toolchain 2').\n - If the regression is on the [blacklist](blacklist.md), skip it.\n - If the breakage is 'obviously deliberate', e.g. a lint changing to deny by\n default, find the original PR and double check it went through a Crater\n run. Skip reporting if so.\n - If the regression is in a dependency, it will have probably caused multiple\n regressions so make sure to deal with the dependency first and then ignore\n any duplicates.\n - If this is not a .1 beta (i.e. it's a second beta run), search for the\n regression already being reported. If it was closed as \"wanted regression\"\n skip reporting, if it was closed as \"fixed\" then reopen with a link to the\n log.\n - Report the regression per the template below:\n\nThis template varies depending on crate source (crates.io or a git repo):\n```\n[CRATENAME-1.0.1](https://crates.io/crates/cratename) regressed from stable to beta - http://cargobomb-reports.../log.txt, cc @AUTHOR\n[AUTHOR/REPO#COMMITISH](https://github.com/author/repo/tree/COMMITISH) regressed from stable to beta - http://cargobomb-reports.../log.txt, cc @AUTHOR\n```\nwhere AUTHOR is the github username of the crate author (may not be available\nif the crate is from crates.io in rare cases). You should also paste a snippet\nof the error in the issue.\n\nWhen in doubt file an issue. It's best to force the Rust devs to\nacknowledge the regression.\n\nIf you are interested in triaging once the issues are raised,\nyou can follow the rough instructions below (to be made clearer):\n\nTo triage the reports I use another sandboxed Rust environment to\nverify the regressions before filing them. Make sure the current\nnightly/beta/stable toolchains are installed.\n\n- Find the git repo. If I can't find it (rare) I just skip the crate.\n- Check out the git repo\n- If the repo has version tags, check out the corresponding version,\n otherwise use master (if master fails to reproduce I will poke around\n the commit history a bit to see if I can pull out a failing revision)\n- Run `cargo +stable test` to verify that stable works.\n - If stable does not work I will run it some more to see if it's a flaky\n test, and add it to the blacklist.\n - I will run `cargo +PREVIOUS_RELEASE test` and see if that fails too,\n and if so move on.\n- Run `cargo +beta test` to verify that it fails. Note that this is checking\n 'beta' even if Crater was against 'nightly'. If that succeeds then\n I move on to `cargo +nightly test`.\n\n## License\n\nMIT / Apache 2.0\n","has_starred","hub_user","immutable_tags_settings",{"_70":11,"_71":72},"enabled","rules",[73],".*","is_automated","permissions",{"_77":19,"_78":11,"_79":11},"read","write","admin","user","storage_size",4919113337,"orgData",{"_85":86,"_87":88,"_89":90,"_91":28,"_92":93,"_94":93,"_95":93,"_96":93,"_97":98,"_99":93,"_100":93,"_101":93,"_102":19},"id","87f897a8663d4137ace233754e6b51c3","uuid","87f897a8-663d-4137-ace2-33754e6b51c3","type","organization","orgname","full_name","","location","company","profile_url","date_joined","2017-07-21T22:47:04.043699Z","gravatar_url","gravatar_email","badge","is_active","recentTags",[105],{"_106":39,"_107":108,"_109":110,"_85":111,"_112":113,"_45":130,"_131":108,"_132":28,"_133":51,"_52":134,"_135":136,"_137":121,"_138":130,"_139":61,"_140":19},"content_type","creator",1971661,"full_size",173792644,16661745,"images",[114],{"_115":116,"_117":118,"_119":93,"_120":121,"_122":123,"_124":125,"_126":93,"_127":-5,"_128":110,"_58":61,"_129":-5},"architecture","amd64","digest","sha256:60668938b11a3f62a6e1c2cbd2c307fdd85fa1375f64a67aa77e8ed2412000b5","features","last_pulled","2026-07-16T12:16:07.656580767Z","last_pushed","2017-12-19T23:53:11Z","os","linux","os_features","os_version","size","variant","2017-12-19T23:53:10.301597Z","last_updater","last_updater_username","media_type","latest","repository",1935958,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_33":150,"_157":158,"_146":167,"_168":11},{"_35":151,"_37":152,"_40":41,"_42":43,"_44":11,"_45":46,"_47":48,"_49":153,"_52":30,"_27":28,"_53":54,"_55":39,"_56":57,"_58":59,"_60":61,"_62":-5,"_63":57,"_64":65,"_66":11,"_67":28,"_68":154,"_74":19,"_75":156,"_80":28,"_81":82},[],[39],[51],{"_70":11,"_71":155},[73],{"_77":19,"_78":11,"_79":11},"tags",{"_159":59,"_160":-5,"_161":-5,"_162":163},"count","next","previous","results",[164],{"_106":39,"_107":108,"_109":110,"_85":111,"_112":165,"_45":130,"_131":108,"_132":28,"_133":51,"_52":134,"_135":136,"_137":121,"_138":130,"_139":61,"_140":19},[166],{"_115":116,"_117":118,"_119":93,"_120":121,"_122":123,"_124":125,"_126":93,"_127":-5,"_128":110,"_58":61,"_129":-5},[],"canDeleteTags","actionData","errors"]

rustlang/crater

Tags for rustlang/crater