diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index c3f1b090b..586298e56 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -1346,16 +1346,33 @@ def artifact_show(app, deps, artifact_remotes, ignore_project_artifact_remotes, @click.option( "--deps", "-d", - default=str(_PipelineSelection.RUN), - show_default=True, + default=None, type=FastEnumType( _PipelineSelection, [_PipelineSelection.RUN, _PipelineSelection.BUILD, _PipelineSelection.NONE, _PipelineSelection.ALL], ), - help="The dependencies to checkout", + help="The dependencies to checkout [default: run]", ) @click.option("--integrate/--no-integrate", default=None, is_flag=True, help="Whether to run integration commands") @click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible") +@click.option( + "--buildtree", + is_flag=True, + help=( + "Checkout the cached build tree of the element instead of the artifact. " + "Will fail if a build tree is not available. " + "pull-buildtrees configuration is needed if the build tree is not available locally." + ), +) +@click.option( + "--buildroot", + is_flag=True, + help=( + "Checkout the cached build root (the whole build sandbox) of the element instead of the artifact. " + "Will fail if a build root is not available. " + "pull-buildtrees configuration is needed if the build root is not available locally." + ), +) @click.option( "--tar", default=None, @@ -1394,6 +1411,8 @@ def artifact_checkout( deps, integrate, hardlinks, + buildtree, + buildroot, tar, compression, directory, @@ -1405,9 +1424,31 @@ def artifact_checkout( When this command is executed from a workspace directory, the default is to checkout the artifact of the workspace element. + + Use the --buildtree option to checkout the build directory of the + element as it was cached at the end of the build, or the --buildroot + option to checkout the whole build sandbox. Build trees and build roots + are only cached according to the cache-buildtrees configuration, and + only build elements have a build tree. """ from .. import utils + if buildtree and buildroot: + click.echo("ERROR: options --buildtree and --buildroot conflict", err=True) + sys.exit(-1) + + if buildtree or buildroot: + option = "--buildtree" if buildtree else "--buildroot" + if deps is not None: + click.echo("ERROR: options --deps and {} conflict".format(option), err=True) + sys.exit(-1) + if integrate is not None: + click.echo("ERROR: options --integrate/--no-integrate and {} conflict".format(option), err=True) + sys.exit(-1) + + if deps is None: + deps = _PipelineSelection.RUN + if hardlinks and tar: click.echo("ERROR: options --hardlinks and --tar conflict", err=True) sys.exit(-1) @@ -1459,6 +1500,8 @@ def artifact_checkout( hardlinks=hardlinks, compression=compression, tar=bool(tar), + buildtree=buildtree, + buildroot=buildroot, artifact_remotes=artifact_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index a1d88d667..7afe7df02 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -679,6 +679,8 @@ def push( # will be placed at the given location. If true and # location is '-', the tarball will be dumped on the # standard output. + # buildtree: Checkout the cached build tree instead of the artifact + # buildroot: Checkout the cached build root instead of the artifact # artifact_remotes: Artifact cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # @@ -693,9 +695,16 @@ def checkout( hardlinks: bool = False, compression: str = "", tar: bool = False, + buildtree: bool = False, + buildroot: bool = False, artifact_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ): + # The build tree and build root are stored in the artifact of the + # target element itself, dependencies are not needed. + assert not (buildtree and buildroot) + if buildtree or buildroot: + selection = _PipelineSelection.NONE elements = self._load( (target,), @@ -721,6 +730,10 @@ def checkout( self.query_cache(elements) self._pull_missing_artifacts(elements) + if buildtree or buildroot: + self._checkout_buildtree(element, location, compression, tar, hardlinks, buildroot=buildroot) + return + try: scope = { _PipelineSelection.RUN: _Scope.RUN, @@ -737,6 +750,75 @@ def checkout( "Error while staging dependencies into a sandbox" ": '{}'".format(e), detail=e.detail, reason=e.reason ) from e + # _checkout_buildtree() + # + # Checkout the build tree or the build root of a cached artifact + # + # Args: + # element (Element/ArtifactElement): The Element/ArtifactElement we want to checkout + # location (str): The name of the directory/the tarfile we want to export to/create + # compression (str): The type of compression for the tarball + # tar (bool): Whether we want to create a tarfile + # hardlinks (bool): Whether to checkout hardlinks instead of copying + # buildroot (bool): Whether to checkout the build root instead of the build tree + # + def _checkout_buildtree(self, element, location, compression, tar, hardlinks, *, buildroot=False): + pull_detail = "pull-buildtrees configuration is needed if the buildtree is not available locally" + + if not element._cached(): + raise StreamError("Artifact not cached locally or in available remotes", reason="missing-artifact") + + if buildroot: + if not element._cached_buildroot(): + if element._buildroot_exists(): + raise StreamError( + "Buildroot is not cached locally or in available remotes", + detail=pull_detail, + reason="missing-buildroot-not-cached", + ) + raise StreamError( + "Artifact was created without buildroot", + detail="Use the cache-buildtrees configuration to cache buildroots when building", + reason="missing-buildroot-artifact-created-without-buildroot", + ) + elif not element._cached_buildtree(): + if element._buildtree_exists(): + raise StreamError( + "Buildtree is not cached locally or in available remotes", + detail=pull_detail, + reason="missing-buildtree-not-cached", + ) + if element._buildroot_exists(): + # Only build elements have a build tree, other elements + # such as script or compose elements only have a build root + raise StreamError( + "Artifact was created without buildtree, but a buildroot is available", + detail="Use the --buildroot option to checkout the buildroot instead", + reason="missing-buildtree-buildroot-available", + ) + raise StreamError( + "Artifact was created without buildtree", + detail="Use the cache-buildtrees configuration to cache buildtrees when building", + reason="missing-buildtree-artifact-created-without-buildtree", + ) + + # Raise warning if the element is cached in a failed state + if element._cached_failure(): + self._context.messenger.warn( + "using a {} from a failed build.".format("buildroot" if buildroot else "buildtree") + ) + + try: + artifact = element._get_artifact() + virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree() + self._export_artifact(tar, location, compression, element, hardlinks, virdir) + except BstError as e: + raise StreamError( + "Error while exporting {}: '{}'".format("buildroot" if buildroot else "buildtree", e), + detail=e.detail, + reason=e.reason, + ) from e + # _export_artifact() # # Export the files of the artifact/a tarball to a virtual directory diff --git a/tests/integration/checkoutbuildtrees.py b/tests/integration/checkoutbuildtrees.py new file mode 100644 index 000000000..199162c15 --- /dev/null +++ b/tests/integration/checkoutbuildtrees.py @@ -0,0 +1,271 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Pylint doesn't play well with fixtures and dependency injection from pytest +# pylint: disable=redefined-outer-name + +import os +import tarfile + +import pytest + +from buildstream._testing import cli, cli_integration, Cli # pylint: disable=unused-import +from buildstream.exceptions import ErrorDomain +from buildstream._testing._utils.site import HAVE_SANDBOX + +from tests.testutils import create_artifact_share + +pytestmark = pytest.mark.integration + + +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") + + +# +# Checkout the buildtree and the buildroot of a build element +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.parametrize("hardlinks", [False, True], ids=["copy", "hardlinks"]) +def test_checkout_buildtree(cli_integration, tmpdir, datafiles, hardlinks): + project = str(datafiles) + element_name = "build-shell/buildtree.bst" + checkout = os.path.join(str(tmpdir), "checkout") + + res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) + res.assert_success() + + args = ["artifact", "checkout", "--buildtree", "--directory", checkout, element_name] + if hardlinks: + args += ["--hardlinks"] + res = cli_integration.run(project=project, args=args) + res.assert_success() + + # Only the content of the build directory is checked out + assert os.listdir(checkout) == ["test"] + with open(os.path.join(checkout, "test"), encoding="utf-8") as f: + assert f.read() == "Hi\n" + + checkout = os.path.join(str(tmpdir), "checkout-buildroot") + args = ["artifact", "checkout", "--buildroot", "--directory", checkout, element_name] + if hardlinks: + args += ["--hardlinks"] + res = cli_integration.run(project=project, args=args) + res.assert_success() + + # The whole build sandbox is checked out, including the build directory + with open( + os.path.join(checkout, "buildstream", "test", "build-shell", "buildtree.bst", "test"), encoding="utf-8" + ) as f: + assert f.read() == "Hi\n" + assert os.path.lexists(os.path.join(checkout, "bin", "sh")) + + +# +# Checkout the buildtree and the buildroot as a tarball +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.parametrize("compression", [None, "gz", "xz", "bz2"]) +def test_checkout_buildtree_tar(cli_integration, tmpdir, datafiles, compression): + project = str(datafiles) + element_name = "build-shell/buildtree.bst" + tar = os.path.join(str(tmpdir), "buildtree.tar") + + res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) + res.assert_success() + + args = ["artifact", "checkout", "--buildtree", "--tar", tar, element_name] + if compression: + args += ["--compression", compression] + res = cli_integration.run(project=project, args=args) + res.assert_success() + + with tarfile.open(tar, "r:" + (compression or "")) as tf: + assert tf.getnames() == [os.path.join(".", "test")] + + tar = os.path.join(str(tmpdir), "buildroot.tar") + args = ["artifact", "checkout", "--buildroot", "--tar", tar, element_name] + if compression: + args += ["--compression", compression] + res = cli_integration.run(project=project, args=args) + res.assert_success() + + with tarfile.open(tar, "r:" + (compression or "")) as tf: + names = tf.getnames() + assert os.path.join(".", "buildstream", "test", "build-shell", "buildtree.bst", "test") in names + assert os.path.join(".", "bin", "sh") in names + + +# +# Checkout the buildtree of a failed build +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_checkout_buildtree_from_failure(cli_integration, tmpdir, datafiles): + project = str(datafiles) + element_name = "build-shell/buildtree-fail.bst" + checkout = os.path.join(str(tmpdir), "checkout") + + res = cli_integration.run(project=project, args=["build", element_name]) + res.assert_main_error(ErrorDomain.STREAM, None) + + res = cli_integration.run( + project=project, args=["artifact", "checkout", "--buildtree", "--directory", checkout, element_name] + ) + res.assert_success() + assert "WARNING using a buildtree from a failed build" in res.stderr + assert os.listdir(checkout) == ["test"] + + +# +# Elements which are not build elements only have a buildroot +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_checkout_buildtree_script_element(cli_integration, tmpdir, datafiles): + project = str(datafiles) + element_name = "build-shell/script.bst" + checkout = os.path.join(str(tmpdir), "checkout") + + res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) + res.assert_success() + + # Requesting the buildtree fails, suggesting to use the buildroot instead + res = cli_integration.run( + project=project, args=["artifact", "checkout", "--buildtree", "--directory", checkout, element_name] + ) + res.assert_main_error(ErrorDomain.STREAM, "missing-buildtree-buildroot-available") + assert "--buildroot" in res.stderr + + res = cli_integration.run( + project=project, args=["artifact", "checkout", "--buildroot", "--directory", checkout, element_name] + ) + res.assert_success() + with open(os.path.join(checkout, "test"), encoding="utf-8") as f: + assert f.read() == "Hi\n" + + +# +# Neither the buildtree nor the buildroot is available when they were not +# cached at build time +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.parametrize( + "option,expect_error", + [ + ("--buildtree", "missing-buildtree-artifact-created-without-buildtree"), + ("--buildroot", "missing-buildroot-artifact-created-without-buildroot"), + ], +) +def test_checkout_buildtree_not_created(cli, tmpdir, datafiles, option, expect_error): + project = str(datafiles) + element_name = "build-shell/buildtree.bst" + checkout = os.path.join(str(tmpdir), "checkout") + + # Use a fresh cache, so that the artifact cannot have been built with + # a buildtree by another test + res = cli.run(project=project, args=["--cache-buildtrees", "never", "build", element_name]) + res.assert_success() + + res = cli.run(project=project, args=["artifact", "checkout", option, "--directory", checkout, element_name]) + res.assert_main_error(ErrorDomain.STREAM, expect_error) + + +# +# The artifact is not cached at all +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("option", ["--buildtree", "--buildroot"]) +def test_checkout_buildtree_artifact_not_cached(cli, tmpdir, datafiles, option): + project = str(datafiles) + element_name = "build-shell/buildtree.bst" + checkout = os.path.join(str(tmpdir), "checkout") + + res = cli.run(project=project, args=["artifact", "checkout", option, "--directory", checkout, element_name]) + res.assert_main_error(ErrorDomain.STREAM, "missing-artifact") + + +# +# The artifact was created with a buildtree, but the buildtree is only +# pulled from the remote with the pull-buildtrees configuration +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.parametrize( + "option,expect_error", + [ + ("--buildtree", "missing-buildtree-not-cached"), + ("--buildroot", "missing-buildroot-not-cached"), + ], +) +def test_checkout_buildtree_pull(cli, tmpdir, datafiles, option, expect_error): + project = str(datafiles) + element_name = "build-shell/buildtree.bst" + checkout = os.path.join(str(tmpdir), "checkout") + + with create_artifact_share(os.path.join(str(tmpdir), "share")) as share: + # Build and push the artifact with its buildtree from a separate local cache + build_cli = Cli(os.path.join(str(tmpdir), "build-cache")) + build_cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) + res = build_cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) + res.assert_success() + assert share.get_artifact(build_cli.get_artifact_name(project, "test", element_name)) + + # The artifact is only available in the share + cli.configure({"artifacts": {"servers": [{"url": share.repo}]}}) + assert cli.get_element_state(project, element_name) != "cached" + + # Without pull-buildtrees, the artifact is pulled but not the buildtree + res = cli.run(project=project, args=["artifact", "checkout", option, "--directory", checkout, element_name]) + res.assert_main_error(ErrorDomain.STREAM, expect_error) + assert "pull-buildtrees" in res.stderr + assert cli.get_element_state(project, element_name) == "cached" + + # With pull-buildtrees, the missing buildtree is pulled + res = cli.run( + project=project, + args=["--pull-buildtrees", "artifact", "checkout", option, "--directory", checkout, element_name], + ) + res.assert_success() + if option == "--buildtree": + assert os.listdir(checkout) == ["test"] + else: + assert os.path.exists( + os.path.join(checkout, "buildstream", "test", "build-shell", "buildtree.bst", "test") + ) + + +# +# Invalid combinations of options +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize( + "args,expect_error", + [ + (["--buildtree", "--buildroot"], "ERROR: options --buildtree and --buildroot conflict"), + (["--buildtree", "--deps", "none"], "ERROR: options --deps and --buildtree conflict"), + (["--buildroot", "--deps", "build"], "ERROR: options --deps and --buildroot conflict"), + (["--buildtree", "--integrate"], "ERROR: options --integrate/--no-integrate and --buildtree conflict"), + (["--buildroot", "--no-integrate"], "ERROR: options --integrate/--no-integrate and --buildroot conflict"), + ], +) +def test_checkout_buildtree_conflicting_options(cli, datafiles, args, expect_error): + project = str(datafiles) + element_name = "build-shell/buildtree.bst" + + res = cli.run(project=project, args=["artifact", "checkout", *args, element_name]) + assert res.exit_code != 0 + assert expect_error in res.stderr