From 3fbb9bd86424581b5a8538a4c53d6e4c03d51547 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 4 Nov 2020 01:47:07 +0100 Subject: [PATCH] Add auto creation of Azure volumes during `compose up` Signed-off-by: Ulysses Souza --- aci/aci.go | 36 +++++++++++++++++++++++++++++++ aci/compose.go | 35 ++++++++++++++++++++++++++++-- aci/convert/volume.go | 23 +++++++++++--------- aci/convert/volume_test.go | 4 ++-- aci/volumes.go | 7 ++++-- tests/aci-e2e/e2e-aci_test.go | 40 +++++++++++++++++------------------ 6 files changed, 109 insertions(+), 36 deletions(-) diff --git a/aci/aci.go b/aci/aci.go index 49eaa7b0c..ecfd6ed91 100644 --- a/aci/aci.go +++ b/aci/aci.go @@ -28,6 +28,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" tm "github.com/buger/goterm" + "github.com/compose-spec/compose-go/types" "github.com/gobwas/ws" "github.com/gobwas/ws/wsutil" "github.com/morikuni/aec" @@ -35,6 +36,7 @@ import ( "github.com/docker/compose-cli/aci/convert" "github.com/docker/compose-cli/aci/login" + "github.com/docker/compose-cli/api/client" "github.com/docker/compose-cli/api/containers" "github.com/docker/compose-cli/context/store" "github.com/docker/compose-cli/errdefs" @@ -64,12 +66,46 @@ func createACIContainers(ctx context.Context, aciContext store.AciContext, group return createOrUpdateACIContainers(ctx, aciContext, groupDefinition) } +func autocreateFileshares(ctx context.Context, project *types.Project) error { + clt, err := client.New(ctx) + if err != nil { + return err + } + for _, v := range project.Volumes { + if v.Driver != convert.AzureFileDriverName { + return fmt.Errorf("cannot use ACI volume, required driver is %q, found %q", convert.AzureFileDriverName, v.Driver) + } + shareName, ok := v.DriverOpts[convert.VolumeDriveroptsShareNameKey] + if !ok { + return fmt.Errorf("cannot retrieve fileshare name for Azure file share") + } + accountName, ok := v.DriverOpts[convert.VolumeDriveroptsAccountNameKey] + if !ok { + return fmt.Errorf("cannot retrieve account name for Azure file share") + } + _, err = clt.VolumeService().Inspect(ctx, fmt.Sprintf("%s/%s", accountName, shareName)) + if err != nil { // Not found, autocreate fileshare + if !errdefs.IsNotFoundError(err) { + return err + } + aciVolumeOpts := &VolumeCreateOptions{ + Account: accountName, + } + if _, err = clt.VolumeService().Create(ctx, shareName, aciVolumeOpts); err != nil { + return err + } + } + } + return nil +} + func createOrUpdateACIContainers(ctx context.Context, aciContext store.AciContext, groupDefinition containerinstance.ContainerGroup) error { w := progress.ContextWriter(ctx) containerGroupsClient, err := login.NewContainerGroupsClient(aciContext.SubscriptionID) if err != nil { return errors.Wrapf(err, "cannot get container group client") } + groupDisplay := "Group " + *groupDefinition.Name w.Event(progress.Event{ ID: groupDisplay, diff --git a/aci/compose.go b/aci/compose.go index 73828e7de..b9a6d1201 100644 --- a/aci/compose.go +++ b/aci/compose.go @@ -46,18 +46,49 @@ func newComposeService(ctx store.AciContext) aciComposeService { func (cs *aciComposeService) Up(ctx context.Context, project *types.Project, detach bool) error { logrus.Debugf("Up on project with name %q", project.Name) - groupDefinition, err := convert.ToContainerGroup(ctx, cs.ctx, *project, cs.storageLogin) - addTag(&groupDefinition, composeContainerTag) + if err := autocreateFileshares(ctx, project); err != nil { + return err + } + + groupDefinition, err := convert.ToContainerGroup(ctx, cs.ctx, *project, cs.storageLogin) if err != nil { return err } + + addTag(&groupDefinition, composeContainerTag) return createOrUpdateACIContainers(ctx, cs.ctx, groupDefinition) } +func (cs aciComposeService) warnKeepVolumeOnDown(ctx context.Context, projectName string) error { + cgClient, err := login.NewContainerGroupsClient(cs.ctx.SubscriptionID) + if err != nil { + return err + } + cg, err := cgClient.Get(ctx, cs.ctx.ResourceGroup, projectName) + if err != nil { + return err + } + if cg.Volumes == nil { + return nil + } + for _, v := range *cg.Volumes { + if v.AzureFile == nil || v.AzureFile.StorageAccountName == nil || v.AzureFile.ShareName == nil { + continue + } + fmt.Printf("WARNING: fileshare \"%s/%s\" will NOT be deleted. Use 'docker volume rm' if you want to delete this volume\n", + *v.AzureFile.StorageAccountName, *v.AzureFile.ShareName) + } + return nil +} + func (cs *aciComposeService) Down(ctx context.Context, project string) error { logrus.Debugf("Down on project with name %q", project) + if err := cs.warnKeepVolumeOnDown(ctx, project); err != nil { + return err + } + cg, err := deleteACIContainerGroup(ctx, cs.ctx, project) if err != nil { return err diff --git a/aci/convert/volume.go b/aci/convert/volume.go index 51b6fd86a..8bb2e9a3b 100644 --- a/aci/convert/volume.go +++ b/aci/convert/volume.go @@ -32,21 +32,24 @@ import ( ) const ( - azureFileDriverName = "azure_file" - volumeDriveroptsShareNameKey = "share_name" - volumeDriveroptsAccountNameKey = "storage_account_name" + // AzureFileDriverName driver name for azure file share + AzureFileDriverName = "azure_file" + // VolumeDriveroptsShareNameKey driver opt for fileshare name + VolumeDriveroptsShareNameKey = "share_name" + // VolumeDriveroptsAccountNameKey driver opt for storage account + VolumeDriveroptsAccountNameKey = "storage_account_name" volumeReadOnly = "read_only" ) func (p projectAciHelper) getAciFileVolumes(ctx context.Context, helper login.StorageLogin) ([]containerinstance.Volume, error) { var azureFileVolumesSlice []containerinstance.Volume for name, v := range p.Volumes { - if v.Driver == azureFileDriverName { - shareName, ok := v.DriverOpts[volumeDriveroptsShareNameKey] + if v.Driver == AzureFileDriverName { + shareName, ok := v.DriverOpts[VolumeDriveroptsShareNameKey] if !ok { return nil, fmt.Errorf("cannot retrieve fileshare name for Azurefile") } - accountName, ok := v.DriverOpts[volumeDriveroptsAccountNameKey] + accountName, ok := v.DriverOpts[VolumeDriveroptsAccountNameKey] if !ok { return nil, fmt.Errorf("cannot retrieve account name for Azurefile") } @@ -105,15 +108,15 @@ func GetRunVolumes(volumes []string) (map[string]types.VolumeConfig, []types.Ser readOnly := strconv.FormatBool(vi.readonly) projectVolumes[vi.name] = types.VolumeConfig{ Name: vi.name, - Driver: azureFileDriverName, + Driver: AzureFileDriverName, DriverOpts: map[string]string{ - volumeDriveroptsAccountNameKey: vi.storageAccount, - volumeDriveroptsShareNameKey: vi.fileshare, + VolumeDriveroptsAccountNameKey: vi.storageAccount, + VolumeDriveroptsShareNameKey: vi.fileshare, volumeReadOnly: readOnly, }, } sv := types.ServiceVolumeConfig{ - Type: azureFileDriverName, + Type: AzureFileDriverName, Source: vi.name, Target: vi.target, ReadOnly: vi.readonly, diff --git a/aci/convert/volume_test.go b/aci/convert/volume_test.go index 36062bf7b..1a8fea972 100644 --- a/aci/convert/volume_test.go +++ b/aci/convert/volume_test.go @@ -207,8 +207,8 @@ func getAzurefileVolumeConfig(name string, accountNameKey string, shareNameKey s Name: name, Driver: "azure_file", DriverOpts: map[string]string{ - volumeDriveroptsAccountNameKey: accountNameKey, - volumeDriveroptsShareNameKey: shareNameKey, + VolumeDriveroptsAccountNameKey: accountNameKey, + VolumeDriveroptsShareNameKey: shareNameKey, volumeReadOnly: strconv.FormatBool(readOnly), }, } diff --git a/aci/volumes.go b/aci/volumes.go index e7e3e0ad8..242e38659 100644 --- a/aci/volumes.go +++ b/aci/volumes.go @@ -232,7 +232,7 @@ func (cs *aciVolumeService) Delete(ctx context.Context, id string, options inter } result, err := fileShareClient.Delete(ctx, cs.aciContext.ResourceGroup, storageAccount, fileshare) - if result.StatusCode == 204 { + if result.HasHTTPStatus(http.StatusNoContent) { return errors.Wrapf(errdefs.ErrNotFound, "fileshare %q", fileshare) } return err @@ -247,8 +247,11 @@ func (cs *aciVolumeService) Inspect(ctx context.Context, id string) (volumes.Vol if err != nil { return volumes.Volume{}, err } - _, err = fileShareClient.Get(ctx, cs.aciContext.ResourceGroup, storageAccount, fileshareName, "") + res, err := fileShareClient.Get(ctx, cs.aciContext.ResourceGroup, storageAccount, fileshareName, "") if err != nil { // Just checks if it exists + if res.HasHTTPStatus(http.StatusNotFound) { + return volumes.Volume{}, errors.Wrapf(errdefs.ErrNotFound, "account %q, file share %q. Original message %s", storageAccount, fileshareName, err.Error()) + } return volumes.Volume{}, err } return toVolume(storageAccount, fileshareName), nil diff --git a/tests/aci-e2e/e2e-aci_test.go b/tests/aci-e2e/e2e-aci_test.go index d3820dcda..8c860a552 100644 --- a/tests/aci-e2e/e2e-aci_test.go +++ b/tests/aci-e2e/e2e-aci_test.go @@ -644,9 +644,7 @@ func TestUpUpdate(t *testing.T) { ) c := NewParallelE2eCLI(t, binDir) sID, groupID, location := setupTestResourceGroup(t, c) - composeAccountName := groupID + "-sa" - composeAccountName = strings.ReplaceAll(composeAccountName, "-", "") - composeAccountName = strings.ToLower(composeAccountName) + composeAccountName := strings.ToLower(strings.ReplaceAll(groupID, "-", "") + "sa") dstDir := filepath.Join(os.TempDir(), "e2e-aci-volume-"+composeAccountName) srcDir := filepath.Join("..", "composefiles", "aci-demo") @@ -663,28 +661,33 @@ func TestUpUpdate(t *testing.T) { multiPortComposefile = filepath.Join(dstDir, multiPortComposefile) volumeID := composeAccountName + "/" + fileshareName + const ( + testFileName = "msg.txt" + testFileContent = "VOLUME_OK" + projectName = "acidemo" + ) + var ( + dnsLabelName = "nginx-" + groupID + fqdn = dnsLabelName + "." + location + ".azurecontainer.io" + ) + t.Run("compose up", func(t *testing.T) { - const ( - testFileName = "msg.txt" - testFileContent = "VOLUME_OK" - projectName = "acidemo" - ) - - c.RunDockerCmd("volume", "create", "--storage-account", composeAccountName, fileshareName) - - // Bootstrap volume aciContext := store.AciContext{ SubscriptionID: sID, Location: location, ResourceGroup: groupID, } - uploadTestFile(t, aciContext, composeAccountName, fileshareName, testFileName, testFileContent) - - dnsLabelName := "nginx-" + groupID - fqdn := dnsLabelName + "." + location + ".azurecontainer.io" - // Name of Compose project is taken from current folder "acie2e" c.RunDockerCmd("compose", "up", "-f", singlePortVolumesComposefile, "--domainname", dnsLabelName, "--project-name", projectName) + // Volume should be autocreated by the "compose up" + uploadTestFile(t, aciContext, composeAccountName, fileshareName, testFileName, testFileContent) + }) + + t.Cleanup(func() { + c.RunDockerCmd("volume", "rm", volumeID) + }) + + t.Run("check deployed compose app", func(t *testing.T) { res := c.RunDockerCmd("ps") out := lines(res.Stdout()) // Check three containers are running @@ -723,9 +726,6 @@ func TestUpUpdate(t *testing.T) { composeAccountName, fileshareName, projectName), }) }) - t.Cleanup(func() { - c.RunDockerCmd("volume", "rm", volumeID) - }) t.Run("compose ps", func(t *testing.T) { res := c.RunDockerCmd("compose", "ps", "--project-name", composeProjectName, "--quiet")