docker
/
setup-buildx-action
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #165 from crazy-max/append 

append nodes to builder support
This commit is contained in:
Tõnis Tiigi 2022-10-14 11:30:50 -07:00 committed by GitHub
parent 95cb08cb26 bd61d52837
commit 59b5ed6124
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 365 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
.github/workflows/ci.yml vendored
View File

@ -305,9 +305,13 @@ jobs:
platforms: ${{ matrix.qemu-platforms }}
-
name: Set up Docker Buildx
id: buildx
uses: ./
with:
version: ${{ matrix.buildx-version }}
-
name: List builder platforms
run: echo ${{ steps.buildx.outputs.platforms }}
build-ref:
runs-on: ubuntu-latest
@ -416,3 +420,47 @@ jobs:
echo "::error::Should have failed"
exit 1
fi
append:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Create dummy contexts
run: |
docker context create ctxbuilder2
docker context create ctxbuilder3
-
name: Set up Docker Buildx
id: buildx
uses: ./
with:
append: |
- name: builder2
endpoint: ctxbuilder2
platforms: linux/amd64
driver-opts:
- image=moby/buildkit:master
- network=host
- endpoint: ctxbuilder3
platforms: linux/arm64
-
name: List builder platforms
run: echo ${{ steps.buildx.outputs.platforms }}
platforms:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
-
name: Set up Docker Buildx
uses: ./
with:
platforms: linux/amd64

44
README.md
View File

@ -21,6 +21,7 @@ ___
* [Usage](#usage)
* [Advanced usage](#advanced-usage)
* [Authentication support](docs/advanced/auth.md)
* [Append additional nodes to the builder](docs/advanced/append-nodes.md)
* [Install by default](docs/advanced/install-default.md)
* [BuildKit daemon configuration](docs/advanced/buildkit-config.md)
* [Standalone mode](docs/advanced/standalone.md)
@ -61,6 +62,7 @@ jobs:
## Advanced usage
* [Authentication support](docs/advanced/auth.md)
* [Append additional nodes to the builder](docs/advanced/append-nodes.md)
* [Install by default](docs/advanced/install-default.md)
* [BuildKit daemon configuration](docs/advanced/buildkit-config.md)
* [Standalone mode](docs/advanced/standalone.md)
@ -69,32 +71,36 @@ jobs:
### inputs
Following inputs can be used as `step.with` keys
Following inputs can be used as `step.with` keys:
| Name | Type | Description |
|-------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `version` | String | [Buildx](https://github.com/docker/buildx) version. (eg. `v0.3.0`, `latest`, `https://github.com/docker/buildx.git#master`) |
| `driver` | String | Sets the [builder driver](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver) to be used (default `docker-container`) |
| `driver-opts` | CSV | List of additional [driver-specific options](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver-opt) (eg. `image=moby/buildkit:master`) |
| `buildkitd-flags` | String | [Flags for buildkitd](https://docs.docker.com/engine/reference/commandline/buildx_create/#buildkitd-flags) daemon (since [buildx v0.3.0](https://github.com/docker/buildx/releases/tag/v0.3.0)) |
| `install` | Bool | Sets up `docker build` command as an alias to `docker buildx` (default `false`) |
| `use` | Bool | Switch to this builder instance (default `true`) |
| `endpoint` | String | [Optional address for docker socket](https://docs.docker.com/engine/reference/commandline/buildx_create/#description) or context from `docker context ls` |
| `config`¹ | String | [BuildKit config file](https://docs.docker.com/engine/reference/commandline/buildx_create/#config) |
| `config-inline`¹ | String | Same as `config` but inline |
> * ¹ `config` and `config-inline` are mutually exclusive
> `CSV` type must be a newline-delimited string
> ```yaml
> driver-opts: image=moby/buildkit:master
> ```
> `List` type is a newline-delimited string
> ```yaml
> driver-opts: |
> image=moby/buildkit:master
> network=host
> ```
> `CSV` type must be a newline-delimited string
> ```yaml
> platforms: linux/amd64,linux/arm64
> ```
| Name | Type | Description |
|-------------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `version` | String | [Buildx](https://github.com/docker/buildx) version. (eg. `v0.3.0`, `latest`, `https://github.com/docker/buildx.git#master`) |
| `driver` | String | Sets the [builder driver](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver) to be used (default `docker-container`) |
| `driver-opts` | List | List of additional [driver-specific options](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver-opt) (eg. `image=moby/buildkit:master`) |
| `buildkitd-flags` | String | [Flags for buildkitd](https://docs.docker.com/engine/reference/commandline/buildx_create/#buildkitd-flags) daemon (since [buildx v0.3.0](https://github.com/docker/buildx/releases/tag/v0.3.0)) |
| `install` | Bool | Sets up `docker build` command as an alias to `docker buildx` (default `false`) |
| `use` | Bool | Switch to this builder instance (default `true`) |
| `endpoint` | String | [Optional address for docker socket](https://docs.docker.com/engine/reference/commandline/buildx_create/#description) or context from `docker context ls` |
| `platforms` | List/CSV | Fixed [platforms](https://docs.docker.com/engine/reference/commandline/buildx_create/#platform) for current node. If not empty, values take priority over the detected ones. |
| `config`¹ | String | [BuildKit config file](https://docs.docker.com/engine/reference/commandline/buildx_create/#config) |
| `config-inline`¹ | String | Same as `config` but inline |
| `append` | YAML | [Append additional nodes](docs/advanced/append-nodes.md) to the builder |
> * ¹ `config` and `config-inline` are mutually exclusive
### outputs
Following outputs are available

70
__tests__/context.test.ts
View File

@ -4,6 +4,7 @@ import * as os from 'os';
import * as path from 'path';
import * as uuid from 'uuid';
import * as context from '../src/context';
import * as nodes from '../src/nodes';
const tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'docker-setup-buildx-')).split(path.sep).join(path.posix.sep);
jest.spyOn(context, 'tmpDir').mockImplementation((): string => {
@ -90,6 +91,24 @@ describe('getCreateArgs', () => {
'tls://foo:1234'
]
],
[
4,
new Map<string, string>([
['driver', 'remote'],
['platforms', 'linux/arm64,linux/arm/v7'],
['endpoint', 'tls://foo:1234'],
['install', 'false'],
['use', 'true'],
]),
[
'create',
'--name', 'builder-9b1deb4d-3b7d-4bad-9bdd-2b0d7b3dcb6d',
'--driver', 'remote',
'--platform', 'linux/arm64,linux/arm/v7',
'--use',
'tls://foo:1234'
]
],
])(
'[%d] given %p as inputs, returns %p',
async (num: number, inputs: Map<string, string>, expected: Array<string>) => {
@ -103,6 +122,57 @@ describe('getCreateArgs', () => {
);
});
describe('getAppendArgs', () => {
beforeEach(() => {
process.env = Object.keys(process.env).reduce((object, key) => {
if (!key.startsWith('INPUT_')) {
object[key] = process.env[key];
}
return object;
}, {});
});
// prettier-ignore
test.each([
[
0,
new Map<string, string>([
['install', 'false'],
['use', 'true'],
]),
{
"name": "aws_graviton2",
"endpoint": "ssh://me@graviton2",
"driver-opts": [
"image=moby/buildkit:latest"
],
"buildkitd-flags": "--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host",
"platforms": "linux/arm64"
},
[
'create',
'--name', 'builder-9b1deb4d-3b7d-4bad-9bdd-2b0d7b3dcb6d',
'--append',
'--node', 'aws_graviton2',
'--driver-opt', 'image=moby/buildkit:latest',
'--buildkitd-flags', '--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host',
'--platform', 'linux/arm64',
'ssh://me@graviton2'
]
]
])(
'[%d] given %p as inputs, returns %p',
async (num: number, inputs: Map<string, string>, node: nodes.Node, expected: Array<string>) => {
inputs.forEach((value: string, name: string) => {
setInput(name, value);
});
const inp = await context.getInputs();
const res = await context.getAppendArgs(inp, node, '0.9.0');
expect(res).toEqual(expected);
}
);
});
describe('getInputList', () => {
it('handles single line correctly', async () => {
await setInput('foo', 'bar');

6
action.yml
View File

@ -32,12 +32,18 @@ inputs:
endpoint:
description: 'Optional address for docker socket or context from `docker context ls`'
required: false
platforms:
description: 'Fixed platforms for current node. If not empty, values take priority over the detected ones'
required: false
config:
description: 'BuildKit config file'
required: false
config-inline:
description: 'Inline BuildKit config'
required: false
append:
description: 'Append additional nodes to the builder'
required: false
outputs:
name:

4
dist/index.js generated vendored
View File

File diff suppressed because one or more lines are too long

2
dist/index.js.map generated vendored
View File

File diff suppressed because one or more lines are too long

50
dist/licenses.txt generated vendored
View File

@ -143,6 +143,31 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
csv-parse
MIT
The MIT License (MIT)
Copyright (c) 2010 Adaltas
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
fs.realpath
ISC
The ISC License
@ -254,6 +279,31 @@ PERFORMANCE OF THIS SOFTWARE.
js-yaml
MIT
(The MIT License)
Copyright (C) 2011-2015 by Vitaly Puzrin
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
lru-cache
ISC
The ISC License

56
docs/advanced/append-nodes.md Normal file
View File

@ -0,0 +1,56 @@
# Append additional nodes to the builder
Buildx also supports running builds on multiple machines. This is useful for
building [multi-platform images](https://docs.docker.com/build/building/multi-platform/)
on native nodes for more complicated cases that are not handled by QEMU and
generally have better performance or for distributing the build across multiple
machines.
You can append nodes to the builder that is going to be created with the
`append` input in the form of a YAML string document to remove limitations
intrinsically linked to GitHub Actions (only string format is handled in the
input fields):
| Name | Type | Description |
|-------------------|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `name` | String | [Name of the node](https://docs.docker.com/engine/reference/commandline/buildx_create/#node). If empty, it is the name of the builder it belongs to, with an index number suffix. This is useful to set it if you want to modify/remove a node in an underlying step of you workflow. |
| `endpoint` | String | [Docker context or endpoint](https://docs.docker.com/engine/reference/commandline/buildx_create/#description) of the node to add to the builder |
| `driver-opts` | List | List of additional [driver-specific options](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver-opt) |
| `buildkitd-flags` | String | [Flags for buildkitd](https://docs.docker.com/engine/reference/commandline/buildx_create/#buildkitd-flags) daemon |
| `platforms` | String | Fixed [platforms](https://docs.docker.com/engine/reference/commandline/buildx_create/#platform) for the node. If not empty, values take priority over the detected ones. |
Here is an example using remote nodes with the [`remote` driver](https://docs.docker.com/build/building/drivers/remote/)
and [TLS authentication](auth.md#tls-authentication):
```yaml
name: ci
on:
push:
jobs:
buildx:
runs-on: ubuntu-latest
steps:
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
driver: remote
endpoint: tcp://oneprovider:1234
append: |
- endpoint: tcp://graviton2:1234
platforms: linux/arm64
- endpoint: tcp://linuxone:1234
platforms: linux/s390x
env:
BUILDER_NODE_0_AUTH_TLS_CACERT: ${{ secrets.ONEPROVIDER_CA }}
BUILDER_NODE_0_AUTH_TLS_CERT: ${{ secrets.ONEPROVIDER_CERT }}
BUILDER_NODE_0_AUTH_TLS_KEY: ${{ secrets.ONEPROVIDER_KEY }}
BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.GRAVITON2_CA }}
BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.GRAVITON2_CERT }}
BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.GRAVITON2_KEY }}
BUILDER_NODE_2_AUTH_TLS_CACERT: ${{ secrets.LINUXONE_CA }}
BUILDER_NODE_2_AUTH_TLS_CERT: ${{ secrets.LINUXONE_CERT }}
BUILDER_NODE_2_AUTH_TLS_KEY: ${{ secrets.LINUXONE_KEY }}
```

5
docs/advanced/auth.md
View File

@ -41,11 +41,6 @@ the node in the list of nodes:
* `BUILDER_NODE_<idx>_AUTH_TLS_CERT`
* `BUILDER_NODE_<idx>_AUTH_TLS_KEY`
> **Note**
>
> The index is always `0` at the moment as we don't support (yet) appending new
> nodes with this action.
```yaml
name: ci

7
jest.config.ts
View File

@ -1,10 +1,13 @@
module.exports = {
clearMocks: true,
moduleFileExtensions: ['js', 'ts'],
setupFiles: ["dotenv/config"],
setupFiles: ['dotenv/config'],
testMatch: ['**/*.test.ts'],
transform: {
'^.+\\.ts$': 'ts-jest'
},
moduleNameMapper: {
'^csv-parse/sync': '<rootDir>/node_modules/csv-parse/dist/cjs/sync.cjs'
},
verbose: true
}
};

2
package.json
View File

@ -31,6 +31,8 @@
"@actions/exec": "^1.1.1",
"@actions/http-client": "^2.0.1",
"@actions/tool-cache": "^2.0.1",
"csv-parse": "^5.1.0",
"js-yaml": "^4.1.0",
"semver": "^7.3.7",
"tmp": "^0.2.1",
"uuid": "^9.0.0"

62
src/context.ts
View File

@ -3,7 +3,9 @@ import * as os from 'os';
import path from 'path';
import * as tmp from 'tmp';
import * as uuid from 'uuid';
import {parse} from 'csv-parse/sync';
import * as buildx from './buildx';
import * as nodes from './nodes';
import * as core from '@actions/core';
let _tmpDir: string;
@ -27,11 +29,13 @@ export interface Inputs {
driver: string;
driverOpts: string[];
buildkitdFlags: string;
platforms: string[];
install: boolean;
use: boolean;
endpoint: string;
config: string;
configInline: string;
append: string;
}
export async function getInputs(): Promise<Inputs> {
@ -41,11 +45,13 @@ export async function getInputs(): Promise<Inputs> {
driver: core.getInput('driver') || 'docker-container',
driverOpts: await getInputList('driver-opts', true),
buildkitdFlags: core.getInput('buildkitd-flags') || '--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host',
platforms: await getInputList('platforms'),
install: core.getBooleanInput('install'),
use: core.getBooleanInput('use'),
endpoint: core.getInput('endpoint'),
config: core.getInput('config'),
configInline: core.getInput('config-inline')
configInline: core.getInput('config-inline'),
append: core.getInput('append')
};
}
@ -63,6 +69,9 @@ export async function getCreateArgs(inputs: Inputs, buildxVersion: string): Prom
args.push('--buildkitd-flags', inputs.buildkitdFlags);
}
}
if (inputs.platforms.length > 0) {
args.push('--platform', inputs.platforms.join(','));
}
if (inputs.use) {
args.push('--use');
}
@ -79,6 +88,28 @@ export async function getCreateArgs(inputs: Inputs, buildxVersion: string): Prom
return args;
}
export async function getAppendArgs(inputs: Inputs, node: nodes.Node, buildxVersion: string): Promise<Array<string>> {
const args: Array<string> = ['create', '--name', inputs.name, '--append'];
if (node.name) {
args.push('--node', node.name);
}
if (node['driver-opts'] && buildx.satisfies(buildxVersion, '>=0.3.0')) {
await asyncForEach(node['driver-opts'], async driverOpt => {
args.push('--driver-opt', driverOpt);
});
if (inputs.driver != 'remote' && node['buildkitd-flags']) {
args.push('--buildkitd-flags', node['buildkitd-flags']);
}
}
if (node.platforms) {
args.push('--platform', node.platforms);
}
if (node.endpoint) {
args.push(node.endpoint);
}
return args;
}
export async function getInspectArgs(inputs: Inputs, buildxVersion: string): Promise<Array<string>> {
const args: Array<string> = ['inspect', '--bootstrap'];
if (buildx.satisfies(buildxVersion, '>=0.4.0')) {
@ -88,14 +119,33 @@ export async function getInspectArgs(inputs: Inputs, buildxVersion: string): Pro
}
export async function getInputList(name: string, ignoreComma?: boolean): Promise<string[]> {
const res: Array<string> = [];
const items = core.getInput(name);
if (items == '') {
return [];
return res;
}
return items
.split(/\r?\n/)
.filter(x => x)
.reduce<string[]>((acc, line) => acc.concat(!ignoreComma ? line.split(',').filter(x => x) : line).map(pat => pat.trim()), []);
const records = parse(items, {
columns: false,
relaxQuotes: true,
comment: '#',
relaxColumnCount: true,
skipEmptyLines: true
});
for (const record of records as Array<string[]>) {
if (record.length == 1) {
res.push(record[0]);
continue;
} else if (!ignoreComma) {
res.push(...record);
continue;
}
res.push(record.join(','));
}
return res.filter(item => item).map(pat => pat.trim());
}
export const asyncForEach = async (array, callback) => {

27
src/main.ts
View File

@ -5,6 +5,7 @@ import * as auth from './auth';
import * as buildx from './buildx';
import * as context from './context';
import * as docker from './docker';
import * as nodes from './nodes';
import * as stateHelper from './state-helper';
import * as util from './util';
import * as core from '@actions/core';
@ -71,6 +72,21 @@ async function run(): Promise<void> {
core.endGroup();
}
if (inputs.append) {
core.startGroup(`Appending node(s) to builder`);
let nodeIndex = 1;
for (const node of nodes.Parse(inputs.append)) {
const authOpts = auth.setCredentials(credsdir, nodeIndex, inputs.driver, node.endpoint || '');
if (authOpts.length > 0) {
node['driver-opts'] = [...(node['driver-opts'] || []), ...authOpts];
}
const appendCmd = buildx.getCommand(await context.getAppendArgs(inputs, node, buildxVersion), standalone);
await exec.exec(appendCmd.commandLine, appendCmd.args);
nodeIndex++;
}
core.endGroup();
}
core.startGroup(`Booting builder`);
const inspectCmd = buildx.getCommand(await context.getInspectArgs(inputs, buildxVersion), standalone);
await exec.exec(inspectCmd.commandLine, inspectCmd.args);
@ -88,9 +104,18 @@ async function run(): Promise<void> {
core.startGroup(`Inspect builder`);
const builder = await buildx.inspect(inputs.name, standalone);
const firstNode = builder.nodes[0];
const reducedPlatforms: Array<string> = [];
for (const node of builder.nodes) {
for (const platform of node.platforms?.split(',') || []) {
if (reducedPlatforms.indexOf(platform) > -1) {
continue;
}
reducedPlatforms.push(platform);
}
}
core.info(JSON.stringify(builder, undefined, 2));
core.setOutput('driver', builder.driver);
core.setOutput('platforms', firstNode.platforms);
core.setOutput('platforms', reducedPlatforms.join(','));
core.setOutput('nodes', JSON.stringify(builder.nodes, undefined, 2));
core.setOutput('endpoint', firstNode.endpoint); // TODO: deprecated, to be removed in a later version
core.setOutput('status', firstNode.status); // TODO: deprecated, to be removed in a later version

13
src/nodes.ts Normal file
View File

@ -0,0 +1,13 @@
import * as yaml from 'js-yaml';
export type Node = {
name?: string;
endpoint?: string;
'driver-opts'?: Array<string>;
'buildkitd-flags'?: string;
platforms?: string;
};
export function Parse(data: string): Node[] {
return yaml.load(data) as Node[];
}

5
yarn.lock
View File

@ -1450,6 +1450,11 @@ cssstyle@^2.3.0:
dependencies:
cssom "~0.3.6"
csv-parse@^5.1.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.3.0.tgz#85cc02fc9d1c89bd1b02e69069c960f8b8064322"
integrity sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ==
data-urls@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b"