docker
/
setup-buildx-action
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

append nodes to builder support 

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
This commit is contained in:
CrazyMax 2022-09-19 11:34:47 +02:00
parent 95cb08cb26
commit 2dfca373f3
No known key found for this signature in database
GPG Key ID: 3248E46B6BB8C7F7
12 changed files with 249 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
.github/workflows/ci.yml vendored
View File

@ -305,9 +305,13 @@ jobs:
platforms: ${{ matrix.qemu-platforms }}
-
name: Set up Docker Buildx
id: buildx
uses: ./
with:
version: ${{ matrix.buildx-version }}
-
name: List builder platforms
run: echo ${{ steps.buildx.outputs.platforms }}
build-ref:
runs-on: ubuntu-latest
@ -416,3 +420,32 @@ jobs:
echo "::error::Should have failed"
exit 1
fi
append:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Create dummy contexts
run: |
docker context create ctxbuilder2
docker context create ctxbuilder3
-
name: Set up Docker Buildx
id: buildx
uses: ./
with:
append: |
- name: builder2
endpoint: ctxbuilder2
platforms: linux/amd64
driver-opts:
- image=moby/buildkit:master
- network=host
- endpoint: ctxbuilder3
platforms: linux/arm64
-
name: List builder platforms
run: echo ${{ steps.buildx.outputs.platforms }}

3
README.md
View File

@ -21,6 +21,7 @@ ___
* [Usage](#usage)
* [Advanced usage](#advanced-usage)
* [Authentication support](docs/advanced/auth.md)
* [Append additional nodes to the builder](docs/advanced/append-nodes.md)
* [Install by default](docs/advanced/install-default.md)
* [BuildKit daemon configuration](docs/advanced/buildkit-config.md)
* [Standalone mode](docs/advanced/standalone.md)
@ -61,6 +62,7 @@ jobs:
## Advanced usage
* [Authentication support](docs/advanced/auth.md)
* [Append additional nodes to the builder](docs/advanced/append-nodes.md)
* [Install by default](docs/advanced/install-default.md)
* [BuildKit daemon configuration](docs/advanced/buildkit-config.md)
* [Standalone mode](docs/advanced/standalone.md)
@ -82,6 +84,7 @@ Following inputs can be used as `step.with` keys
| `endpoint` | String | [Optional address for docker socket](https://docs.docker.com/engine/reference/commandline/buildx_create/#description) or context from `docker context ls` |
| `config`¹ | String | [BuildKit config file](https://docs.docker.com/engine/reference/commandline/buildx_create/#config) |
| `config-inline`¹ | String | Same as `config` but inline |
| `append` | YAML | [Append additional nodes](docs/advanced/append-nodes.md) to the builder |
> * ¹ `config` and `config-inline` are mutually exclusive

52
__tests__/context.test.ts
View File

@ -4,6 +4,7 @@ import * as os from 'os';
import * as path from 'path';
import * as uuid from 'uuid';
import * as context from '../src/context';
import * as nodes from '../src/nodes';
const tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'docker-setup-buildx-')).split(path.sep).join(path.posix.sep);
jest.spyOn(context, 'tmpDir').mockImplementation((): string => {
@ -103,6 +104,57 @@ describe('getCreateArgs', () => {
);
});
describe('getAppendArgs', () => {
beforeEach(() => {
process.env = Object.keys(process.env).reduce((object, key) => {
if (!key.startsWith('INPUT_')) {
object[key] = process.env[key];
}
return object;
}, {});
});
// prettier-ignore
test.each([
[
0,
new Map<string, string>([
['install', 'false'],
['use', 'true'],
]),
{
"name": "aws_graviton2",
"endpoint": "ssh://me@graviton2",
"driver-opts": [
"image=moby/buildkit:latest"
],
"buildkitd-flags": "--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host",
"platforms": "linux/arm64"
},
[
'create',
'--name', 'builder-9b1deb4d-3b7d-4bad-9bdd-2b0d7b3dcb6d',
'--append',
'--node', 'aws_graviton2',
'--driver-opt', 'image=moby/buildkit:latest',
'--buildkitd-flags', '--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host',
'--platform', 'linux/arm64',
'ssh://me@graviton2'
]
]
])(
'[%d] given %p as inputs, returns %p',
async (num: number, inputs: Map<string, string>, node: nodes.Node, expected: Array<string>) => {
inputs.forEach((value: string, name: string) => {
setInput(name, value);
});
const inp = await context.getInputs();
const res = await context.getAppendArgs(inp, node, '0.9.0');
expect(res).toEqual(expected);
}
);
});
describe('getInputList', () => {
it('handles single line correctly', async () => {
await setInput('foo', 'bar');

3
action.yml
View File

@ -38,6 +38,9 @@ inputs:
config-inline:
description: 'Inline BuildKit config'
required: false
append:
description: 'Append additional nodes to the builder'
required: false
outputs:
name:

56
docs/advanced/append-nodes.md Normal file
View File

@ -0,0 +1,56 @@
# Append additional nodes to the builder
Buildx also supports running builds on multiple machines. This is useful for
building [multi-platform images](https://docs.docker.com/build/building/multi-platform/)
on native nodes for more complicated cases that are not handled by QEMU and
generally have better performance or for distributing the build across multiple
machines.
You can append nodes to the builder that is going to be created with the
`append` input in the form of a YAML string document to remove limitations
intrinsically linked to GitHub Actions (only string format is handled in the
input fields):
| Name | Type | Description |
|-------------------|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `name` | String | [Name of the node](https://docs.docker.com/engine/reference/commandline/buildx_create/#node). If empty, it is the name of the builder it belongs to, with an index number suffix. This is useful to set it if you want to modify/remove a node in an underlying step of you workflow. |
| `endpoint` | String | [Docker context or endpoint](https://docs.docker.com/engine/reference/commandline/buildx_create/#description) of the node to add to the builder |
| `driver-opts` | List | List of additional [driver-specific options](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver-opt) |
| `buildkitd-flags` | String | [Flags for buildkitd](https://docs.docker.com/engine/reference/commandline/buildx_create/#buildkitd-flags) daemon |
| `platforms` | String | Fixed [platforms](https://docs.docker.com/engine/reference/commandline/buildx_create/#platform) for the node. If not empty, values take priority over the detected ones. |
Here is an example using remote nodes with the [`remote` driver](https://docs.docker.com/build/building/drivers/remote/)
and [TLS authentication](auth.md#tls-authentication):
```yaml
name: ci
on:
push:
jobs:
buildx:
runs-on: ubuntu-latest
steps:
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
driver: remote
endpoint: tcp://oneprovider:1234
append: |
- endpoint: tcp://graviton2:1234
platforms: linux/arm64
- endpoint: tcp://linuxone:1234
platforms: linux/s390x
env:
BUILDER_NODE_0_AUTH_TLS_CACERT: ${{ secrets.ONEPROVIDER_CA }}
BUILDER_NODE_0_AUTH_TLS_CERT: ${{ secrets.ONEPROVIDER_CERT }}
BUILDER_NODE_0_AUTH_TLS_KEY: ${{ secrets.ONEPROVIDER_KEY }}
BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.GRAVITON2_CA }}
BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.GRAVITON2_CERT }}
BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.GRAVITON2_KEY }}
BUILDER_NODE_2_AUTH_TLS_CACERT: ${{ secrets.LINUXONE_CA }}
BUILDER_NODE_2_AUTH_TLS_CERT: ${{ secrets.LINUXONE_CERT }}
BUILDER_NODE_2_AUTH_TLS_KEY: ${{ secrets.LINUXONE_KEY }}
```

5
docs/advanced/auth.md
View File

@ -41,11 +41,6 @@ the node in the list of nodes:
* `BUILDER_NODE_<idx>_AUTH_TLS_CERT`
* `BUILDER_NODE_<idx>_AUTH_TLS_KEY`
> **Note**
>
> The index is always `0` at the moment as we don't support (yet) appending new
> nodes with this action.
```yaml
name: ci

7
jest.config.ts
View File

@ -1,10 +1,13 @@
module.exports = {
clearMocks: true,
moduleFileExtensions: ['js', 'ts'],
setupFiles: ["dotenv/config"],
setupFiles: ['dotenv/config'],
testMatch: ['**/*.test.ts'],
transform: {
'^.+\\.ts$': 'ts-jest'
},
moduleNameMapper: {
'^csv-parse/sync': '<rootDir>/node_modules/csv-parse/dist/cjs/sync.cjs'
},
verbose: true
}
};

2
package.json
View File

@ -31,6 +31,8 @@
"@actions/exec": "^1.1.1",
"@actions/http-client": "^2.0.1",
"@actions/tool-cache": "^2.0.1",
"csv-parse": "^5.1.0",
"js-yaml": "^4.1.0",
"semver": "^7.3.7",
"tmp": "^0.2.1",
"uuid": "^9.0.0"

57
src/context.ts
View File

@ -3,7 +3,9 @@ import * as os from 'os';
import path from 'path';
import * as tmp from 'tmp';
import * as uuid from 'uuid';
import {parse} from 'csv-parse/sync';
import * as buildx from './buildx';
import * as nodes from './nodes';
import * as core from '@actions/core';
let _tmpDir: string;
@ -32,6 +34,7 @@ export interface Inputs {
endpoint: string;
config: string;
configInline: string;
append: string;
}
export async function getInputs(): Promise<Inputs> {
@ -45,7 +48,8 @@ export async function getInputs(): Promise<Inputs> {
use: core.getBooleanInput('use'),
endpoint: core.getInput('endpoint'),
config: core.getInput('config'),
configInline: core.getInput('config-inline')
configInline: core.getInput('config-inline'),
append: core.getInput('append')
};
}
@ -79,6 +83,28 @@ export async function getCreateArgs(inputs: Inputs, buildxVersion: string): Prom
return args;
}
export async function getAppendArgs(inputs: Inputs, node: nodes.Node, buildxVersion: string): Promise<Array<string>> {
const args: Array<string> = ['create', '--name', inputs.name, '--append'];
if (node.name) {
args.push('--node', node.name);
}
if (node['driver-opts'] && buildx.satisfies(buildxVersion, '>=0.3.0')) {
await asyncForEach(node['driver-opts'], async driverOpt => {
args.push('--driver-opt', driverOpt);
});
if (inputs.driver != 'remote' && node['buildkitd-flags']) {
args.push('--buildkitd-flags', node['buildkitd-flags']);
}
}
if (node.platforms) {
args.push('--platform', node.platforms);
}
if (node.endpoint) {
args.push(node.endpoint);
}
return args;
}
export async function getInspectArgs(inputs: Inputs, buildxVersion: string): Promise<Array<string>> {
const args: Array<string> = ['inspect', '--bootstrap'];
if (buildx.satisfies(buildxVersion, '>=0.4.0')) {
@ -88,14 +114,33 @@ export async function getInspectArgs(inputs: Inputs, buildxVersion: string): Pro
}
export async function getInputList(name: string, ignoreComma?: boolean): Promise<string[]> {
const res: Array<string> = [];
const items = core.getInput(name);
if (items == '') {
return [];
return res;
}
return items
.split(/\r?\n/)
.filter(x => x)
.reduce<string[]>((acc, line) => acc.concat(!ignoreComma ? line.split(',').filter(x => x) : line).map(pat => pat.trim()), []);
const records = parse(items, {
columns: false,
relaxQuotes: true,
comment: '#',
relaxColumnCount: true,
skipEmptyLines: true
});
for (const record of records as Array<string[]>) {
if (record.length == 1) {
res.push(record[0]);
continue;
} else if (!ignoreComma) {
res.push(...record);
continue;
}
res.push(record.join(','));
}
return res.filter(item => item).map(pat => pat.trim());
}
export const asyncForEach = async (array, callback) => {

27
src/main.ts
View File

@ -5,6 +5,7 @@ import * as auth from './auth';
import * as buildx from './buildx';
import * as context from './context';
import * as docker from './docker';
import * as nodes from './nodes';
import * as stateHelper from './state-helper';
import * as util from './util';
import * as core from '@actions/core';
@ -71,6 +72,21 @@ async function run(): Promise<void> {
core.endGroup();
}
if (inputs.append) {
core.startGroup(`Appending node(s) to builder`);
let nodeIndex = 1;
for (const node of nodes.Parse(inputs.append)) {
const authOpts = auth.setCredentials(credsdir, nodeIndex, inputs.driver, node.endpoint || '');
if (authOpts.length > 0) {
node['driver-opts'] = [...(node['driver-opts'] || []), ...authOpts];
}
const appendCmd = buildx.getCommand(await context.getAppendArgs(inputs, node, buildxVersion), standalone);
await exec.exec(appendCmd.commandLine, appendCmd.args);
nodeIndex++;
}
core.endGroup();
}
core.startGroup(`Booting builder`);
const inspectCmd = buildx.getCommand(await context.getInspectArgs(inputs, buildxVersion), standalone);
await exec.exec(inspectCmd.commandLine, inspectCmd.args);
@ -88,9 +104,18 @@ async function run(): Promise<void> {
core.startGroup(`Inspect builder`);
const builder = await buildx.inspect(inputs.name, standalone);
const firstNode = builder.nodes[0];
const reducedPlatforms: Array<string> = [];
for (const node of builder.nodes) {
for (const platform of node.platforms?.split(',') || []) {
if (reducedPlatforms.indexOf(platform) > -1) {
continue;
}
reducedPlatforms.push(platform);
}
}
core.info(JSON.stringify(builder, undefined, 2));
core.setOutput('driver', builder.driver);
core.setOutput('platforms', firstNode.platforms);
core.setOutput('platforms', reducedPlatforms.join(','));
core.setOutput('nodes', JSON.stringify(builder.nodes, undefined, 2));
core.setOutput('endpoint', firstNode.endpoint); // TODO: deprecated, to be removed in a later version
core.setOutput('status', firstNode.status); // TODO: deprecated, to be removed in a later version

13
src/nodes.ts Normal file
View File

@ -0,0 +1,13 @@
import * as yaml from 'js-yaml';
export type Node = {
name?: string;
endpoint?: string;
'driver-opts'?: Array<string>;
'buildkitd-flags'?: string;
platforms?: string;
};
export function Parse(data: string): Node[] {
return yaml.load(data) as Node[];
}

5
yarn.lock
View File

@ -1450,6 +1450,11 @@ cssstyle@^2.3.0:
dependencies:
cssom "~0.3.6"
csv-parse@^5.1.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.3.0.tgz#85cc02fc9d1c89bd1b02e69069c960f8b8064322"
integrity sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ==
data-urls@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b"