Fix filename during extracton from docker uri 35/3135/3
authorSerban Jora <sj2381@att.com>
Wed, 10 Oct 2018 19:43:15 +0000 (15:43 -0400)
committerSerban Jora <sj2381@att.com>
Thu, 11 Oct 2018 15:56:54 +0000 (11:56 -0400)
Change-Id: I88943d6a1597ff1a265d5e6670981fadb6df72af
Issue-ID: ACUMOS-1780
Signed-off-by: Serban Jora <sj2381@att.com>
gateway/src/main/java/org/acumos/federation/gateway/adapter/PeerGateway.java
gateway/src/main/java/org/acumos/federation/gateway/cds/Artifact.java
gateway/src/main/java/org/acumos/federation/gateway/cds/Document.java
gateway/src/main/java/org/acumos/federation/gateway/cds/Mapper.java
gateway/src/main/java/org/acumos/federation/gateway/cds/RevisionDescription.java [new file with mode: 0644]
gateway/src/main/java/org/acumos/federation/gateway/cds/TimestampedEntity.java [new file with mode: 0644]
gateway/src/main/java/org/acumos/federation/gateway/service/impl/ContentServiceImpl.java
gateway/src/main/java/org/acumos/federation/gateway/task/PeerSubscriptionTask.java

index 44335e6..d05b69d 100644 (file)
@@ -28,6 +28,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Date;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
@@ -48,6 +49,7 @@ import org.acumos.federation.gateway.cds.Solution;
 import org.acumos.federation.gateway.cds.SolutionRevision;
 import org.acumos.federation.gateway.cds.SubscriptionScope;
 import org.acumos.federation.gateway.cds.PeerSubscription;
+import org.acumos.federation.gateway.cds.TimestampedEntity;
 import org.acumos.federation.gateway.common.Clients;
 import org.acumos.federation.gateway.common.FederationClient;
 import org.acumos.federation.gateway.config.EELFLoggerDelegate;
@@ -55,6 +57,7 @@ import org.acumos.federation.gateway.config.GatewayCondition;
 import org.acumos.federation.gateway.event.PeerSubscriptionEvent;
 import org.acumos.federation.gateway.service.CatalogService;
 import org.acumos.federation.gateway.service.ContentService;
+import org.acumos.federation.gateway.service.PeerSubscriptionService;
 import org.acumos.federation.gateway.service.ServiceContext;
 import org.acumos.federation.gateway.service.ServiceException;
 import org.acumos.federation.gateway.service.impl.AbstractServiceImpl;
@@ -86,6 +89,8 @@ public class PeerGateway {
        private ContentService content;
        @Autowired
        private CatalogService catalog;
+       @Autowired
+       private PeerSubscriptionService peerSubscriptionService;
 
 
        public PeerGateway() {
@@ -150,24 +155,40 @@ public class PeerGateway {
                        this.peer = thePeer;
                        this.sub = new PeerSubscription(theSub);
                        this.solutions = theSolutions;
+               
+                       //remember when we processed the subscription
+                       this.sub.setProcessed(new Date());
                }
 
                public void run() {
 
                        log.info(EELFLoggerDelegate.debugLogger, "Received peer " + this.peer + " solutions: " + this.solutions);
                        ServiceContext ctx = catalog.selfService();
+                       boolean isComplete = true;
 
                        for (MLPSolution peerSolution : this.solutions) {
                                log.info(EELFLoggerDelegate.debugLogger, "Processing peer solution {}", peerSolution);
 
                                try {
-                                       mapSolution(peerSolution, ctx);
+                                       isComplete &= mapSolution(peerSolution, ctx);
                                }
                                catch (Throwable t) {
                                        log.error(EELFLoggerDelegate.errorLogger,
                                                        "Mapping of acumos solution failed for " + peerSolution, t);
                                }
                        }
+                                       
+                       log.info(EELFLoggerDelegate.debugLogger, "Processing of subscription {} completed succesfully: {}", this.sub, isComplete);
+                       //only commit the last processed date if we completed succesfully
+                       if (isComplete) {
+                               try {
+                                       peerSubscriptionService.updatePeerSubscription(this.sub);
+                               }
+                               catch (ServiceException sx) {
+                                       log.error(EELFLoggerDelegate.errorLogger,
+                                                       "Failed to update subscription information", sx);
+                               }
+                       }
                }
 
                //this should go away once the move to service interface based operations is complete
@@ -180,8 +201,8 @@ public class PeerGateway {
                private Artifact copyArtifact(Artifact peerArtifact) {
                        return Artifact.buildFrom(peerArtifact)
                                                                .withUser(getUserId(this.sub))
-                                                               .withCreated(0)
-                                                               .withModified(0)
+                                                               .withCreatedDate(TimestampedEntity.ORIGIN)
+                                                               .withModifiedDate(TimestampedEntity.ORIGIN)
                                                                .build();
                }
 
@@ -222,8 +243,8 @@ public class PeerGateway {
                private Document copyDocument(Document peerDocument) {
                        return Document.buildFrom(peerDocument)
                                                                .withUser(getUserId(this.sub))
-                                                               .withCreated(0)
-                                                               .withModified(0)
+                                                               .withCreatedDate(TimestampedEntity.ORIGIN)
+                                                               .withModifiedDate(TimestampedEntity.ORIGIN)
                                                                .build();
                }
 
@@ -258,7 +279,41 @@ public class PeerGateway {
                                throw new ServiceException("Document handling unexpected failure", x);
                        }
                }
+       
+               private MLPRevisionDescription copyRevisionDescription(MLPRevisionDescription peerDescription) {
+                       MLPRevisionDescription localDescription = new MLPRevisionDescription(peerDescription);
+                       localDescription.setCreated(TimestampedEntity.ORIGIN);
+                       localDescription.setModified(TimestampedEntity.ORIGIN);
+                       return localDescription;
+               }
+
+               private MLPRevisionDescription copyRevisionDescription(MLPRevisionDescription peerDescription, MLPRevisionDescription localDescription) {
+                       localDescription.setDescription(peerDescription.getDescription());
+                       return localDescription;
+               }
 
+               private void putRevisionDescription(MLPRevisionDescription theDescription,ServiceContext theContext) throws ServiceException {
+                       
+                       try {
+                               if (theDescription.getCreated().getTime() == 0) {
+                                       getCDSClient(theContext).createRevisionDescription(theDescription);
+                                       log.info(EELFLoggerDelegate.debugLogger, "Local description created: {}", theDescription);
+                               }
+                               else {
+                                       getCDSClient(theContext).updateRevisionDescription(theDescription);
+                               }
+                       }
+                       catch (HttpStatusCodeException restx) {
+                               log.error(EELFLoggerDelegate.errorLogger,
+                                       "Revision description CDS call failed. CDS message is " + restx.getResponseBodyAsString(), restx);
+                               throw new ServiceException("Revision description CDS call failed.", restx);
+                       }
+                       catch (Exception x) {
+                               log.error(EELFLoggerDelegate.errorLogger, "Revision description handling unexpected failure", x);
+                               throw new ServiceException("Revision description handling unexpected failure", x);
+                       }
+               }
+       
 
                /**
                 * Here comes the core process of updating a local solution's related
@@ -269,11 +324,13 @@ public class PeerGateway {
                 *            artifacts) we are trying to sync
                 * @param theContext
                 *            the context in which we perform the catalog operations
+                * @return true if mapping was succesful, false otherwise
                 * @throws Exception
                 *             any error related to CDS and peer interaction
                 */
-               protected void mapSolution(MLPSolution theSolution, ServiceContext theContext) throws Exception {
+               protected boolean mapSolution(MLPSolution theSolution, ServiceContext theContext) throws Exception {
 
+                       boolean isComplete = true;
                        FederationClient fedClient = clients.getFederationClient(this.peer.getApiUrl());
 
                        Solution localSolution = null,
@@ -293,7 +350,7 @@ public class PeerGateway {
                        // revision (but that's an assumption on how on-boarding works)
                        if (peerRevisions == null || peerRevisions.size() == 0) {
                                log.warn(EELFLoggerDelegate.debugLogger, "No peer revisions were retrieved");
-                               return;
+                               return true;
                        }
 
                        // check if we have locally the latest revision available on the peer
@@ -331,7 +388,8 @@ public class PeerGateway {
                                }
                                catch (Exception x) {
                                        log.error(EELFLoggerDelegate.errorLogger, "Failed to retrieve peer acumos artifact details", x);
-                                       continue; //try procecssing the next revision
+                                       isComplete = false; //try procecessing the next revision but mark the processing as incomplete
+                                       continue;
                                }
 
                                try {
@@ -346,7 +404,8 @@ public class PeerGateway {
                                catch (ServiceException sx) {
                                        log.error(EELFLoggerDelegate.errorLogger,
                                                        "Failed to put revision " + theSolution.getSolutionId() + "/" + peerRevision.getRevisionId() + " into catalog", sx);
-                                       continue; //try procecssing the next revision
+                                       isComplete = false; //try procecessing the next revision but mark the processing as incomplete
+                                       continue;
                                }
 
                                List<Artifact> peerArtifacts = (List)((SolutionRevision)peerRevision).getArtifacts();
@@ -397,6 +456,7 @@ public class PeerGateway {
                                                catch (Exception x) {
                                                        log.error(EELFLoggerDelegate.errorLogger, "Failed to retrieve acumos artifact content", x);
                                                        doCatalog = this.sub.getSubscriptionOptions().alwaysUpdateCatalog();
+                                                       isComplete = false;
                                                }
 
                                                if (artifactContent != null) {
@@ -409,6 +469,7 @@ public class PeerGateway {
                                                                log.error(EELFLoggerDelegate.errorLogger,
                                                                                        "Failed to store artifact content to local repo", sx);
                                                                doCatalog = this.sub.getSubscriptionOptions().alwaysUpdateCatalog();
+                                                               isComplete = false;
                                                        }
                                                        finally {
                                                                if (artifactContent instanceof Closeable) {
@@ -424,6 +485,7 @@ public class PeerGateway {
                                                }
                                                catch (ServiceException sx) {
                                                        log.error(EELFLoggerDelegate.errorLogger, "Artifact processing failed.", sx);
+                                                       isComplete = false;
                                                }
                                        }
                                } //end map artifacts loop
@@ -473,6 +535,7 @@ public class PeerGateway {
                                                catch (Exception x) {
                                                        log.error(EELFLoggerDelegate.errorLogger, "Failed to retrieve acumos document content", x);
                                                        doCatalog = this.sub.getSubscriptionOptions().alwaysUpdateCatalog();
+                                                       isComplete = false;
                                                }
 
                                                if (documentContent != null) {
@@ -485,6 +548,7 @@ public class PeerGateway {
                                                                log.error(EELFLoggerDelegate.errorLogger,
                                                                                        "Failed to store document content to local repo", sx);
                                                                doCatalog = this.sub.getSubscriptionOptions().alwaysUpdateCatalog();
+                                                               isComplete = false;
                                                        }
                                                }
                                        }
@@ -495,33 +559,44 @@ public class PeerGateway {
                                                }
                                                catch (ServiceException sx) {
                                                        log.error(EELFLoggerDelegate.errorLogger,       "Document processing failed",   sx);
+                                                       isComplete = false;
                                                }
                                        }
        
                                } // end map documents loop
                                
-                               MLPRevisionDescription catalogDescription = ((SolutionRevision)localRevision).getRevisionDescription();
+                               MLPRevisionDescription localDescription = ((SolutionRevision)localRevision).getRevisionDescription();
                                MLPRevisionDescription peerDescription = ((SolutionRevision)peerRevision).getRevisionDescription();
 
                                if (peerDescription != null) {
-                                       try {
-                                               if (catalogDescription == null) {
-                                                       getCDSClient(theContext).createRevisionDescription(peerDescription);
-                                               }
-                                               else {
-                                                       //is this a good enough test ??
-                                                       if (peerDescription.getModified().after(catalogDescription.getModified())) {
-                                                               getCDSClient(theContext).updateRevisionDescription(peerDescription);
-                                                       }
+                                       boolean doCatalog = false;
+
+                                       if (localDescription == null) {
+                                               localDescription = copyRevisionDescription(peerDescription);
+                                               doCatalog = true;
+                                       }
+                                       else {
+                                               //is this a good enough test ?? it implies time sync ..
+                                               if (peerDescription.getModified().after(localDescription.getModified())) {
+                                                       localDescription = copyRevisionDescription(peerDescription, localDescription);
+                                                       doCatalog = true;
                                                }
                                        }
-                                       catch (HttpStatusCodeException restx) {
-                                               log.error(EELFLoggerDelegate.errorLogger,
-                                                                       "Failed to store revision description. CDS message is " + restx.getResponseBodyAsString(),
-                                               restx);
+
+                                       if (doCatalog) {
+                                               try {
+                                                       putRevisionDescription(localDescription, theContext);
+                                               }
+                                               catch (ServiceException sx) {
+                                                       log.error(EELFLoggerDelegate.errorLogger,       "Description processing failed",        sx);
+                                                       isComplete = false;
+                                               }
                                        }
                                }
+
                        }
+
+                       return isComplete;
                } // mapSolution
        }
 }
index a22fb9f..9909432 100644 (file)
@@ -28,7 +28,7 @@ import com.github.dockerjava.api.model.Identifier;
 /**
  */
 public class Artifact extends MLPArtifact
-                                                                                       implements Reference {
+                                                                                       implements Reference, TimestampedEntity {
 
        private String filename;
 
@@ -62,13 +62,14 @@ public class Artifact extends MLPArtifact
 
        @JsonIgnore
        @Override
-       public String getCanonicalFilename() {
+       public String getUriFilename() {
                if (ArtifactType.DockerImage == ArtifactType.forCode(getArtifactTypeCode())) {
                        return Identifier.fromCompoundString(getUri()).repository.getPath();
                }
                else {
-                       return Reference.super.getCanonicalFilename();
+                       return Reference.super.getUriFilename();
                }
        }
+
 }
 
index a8b4889..811cb93 100644 (file)
@@ -25,7 +25,7 @@ import org.acumos.cds.domain.MLPDocument;
 /**
  */
 public class Document extends MLPDocument 
-                                                                                       implements Reference {
+                                                                                       implements Reference, TimestampedEntity {
 
        private String filename;
 
index aa6596f..b640b05 100644 (file)
@@ -26,6 +26,7 @@ import org.acumos.cds.domain.MLPDocument;
 import org.acumos.cds.domain.MLPSolution;
 import org.acumos.cds.domain.MLPSolutionRevision;
 import org.acumos.cds.domain.MLPPeerSubscription;
+import org.acumos.cds.domain.MLPRevisionDescription;
 
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -54,6 +55,7 @@ public class Mapper {
     fedModule.addDeserializer(MLPArtifact.class, new ArtifactDeserializer());
     fedModule.addDeserializer(MLPDocument.class, new DocumentDeserializer());
     fedModule.addDeserializer(MLPPeerSubscription.class, new PeerSubscriptionDeserializer());
+    fedModule.addDeserializer(MLPRevisionDescription.class, new RevisionDescriptionDeserializer());
 
                mapper.registerModule(fedModule);
 
@@ -130,5 +132,18 @@ public class Mapper {
        }
        }
 
+       private static class RevisionDescriptionDeserializer extends StdDeserializer<MLPRevisionDescription> {
+               public RevisionDescriptionDeserializer() {
+                       super(MLPRevisionDescription.class);
+               }
+               @Override
+       public MLPRevisionDescription deserialize(JsonParser theParser, DeserializationContext theCtx) 
+                                                                                                                                                                                               throws IOException, JsonProcessingException {
+         ObjectMapper mapper = (ObjectMapper) theParser.getCodec();
+       return mapper.readValue(theParser, RevisionDescription.class);
+       }
+       }
 }
 
diff --git a/gateway/src/main/java/org/acumos/federation/gateway/cds/RevisionDescription.java b/gateway/src/main/java/org/acumos/federation/gateway/cds/RevisionDescription.java
new file mode 100644 (file)
index 0000000..ccc6b33
--- /dev/null
@@ -0,0 +1,32 @@
+/*-
+ * ===============LICENSE_START=======================================================
+ * Acumos
+ * ===================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property & Tech Mahindra. All rights reserved.
+ * ===================================================================================
+ * This Acumos software file is distributed by AT&T and Tech Mahindra
+ * under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *  
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ===============LICENSE_END=========================================================
+ */
+package org.acumos.federation.gateway.cds;
+
+
+import org.acumos.cds.domain.MLPRevisionDescription;
+
+/**
+ */
+public class RevisionDescription extends MLPRevisionDescription
+                                                                                                                                       implements TimestampedEntity {
+
+}
+
+
diff --git a/gateway/src/main/java/org/acumos/federation/gateway/cds/TimestampedEntity.java b/gateway/src/main/java/org/acumos/federation/gateway/cds/TimestampedEntity.java
new file mode 100644 (file)
index 0000000..34b85b2
--- /dev/null
@@ -0,0 +1,52 @@
+/*-
+ * ===============LICENSE_START=======================================================
+ * Acumos
+ * ===================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property & Tech Mahindra. All rights reserved.
+ * ===================================================================================
+ * This Acumos software file is distributed by AT&T and Tech Mahindra
+ * under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *  
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ===============LICENSE_END=========================================================
+ */
+package org.acumos.federation.gateway.cds;
+
+import java.util.Date;
+
+
+/**
+ */
+public interface TimestampedEntity {
+
+       public static final Date ORIGIN = new Date(0);
+
+       public Date getCreated();
+
+       public void setCreated(Date created); 
+
+       public default void resetCreated() {
+               setCreated(ORIGIN);
+       }
+
+       public Date getModified(); 
+
+       public void setModified(Date modified); 
+
+       public default void resetModified() {
+               setModified(ORIGIN);
+       }
+       
+       public default void resetTimestamp() {
+               resetCreated();
+               resetModified();
+       }
+}
+
index 749263f..23f3ef4 100644 (file)
@@ -47,6 +47,7 @@ import org.acumos.federation.gateway.service.ServiceException;
 import org.springframework.http.MediaType;
 import org.springframework.http.RequestEntity;
 import org.springframework.http.ResponseEntity;
+import org.springframework.web.util.UriUtils;
 import org.springframework.web.client.HttpStatusCodeException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.Resource;
@@ -195,7 +196,7 @@ public class ContentServiceImpl extends AbstractServiceImpl
                        }
                }
                else {
-                       String[] nameParts = splitName(theArtifact.getFilename());
+                       String[] nameParts = splitName(getName(theArtifact));
                        String uri = putNexusContent(
                                nexusPrefix(theSolutionId, theRevisionId), nameParts[0], theArtifact.getVersion(), nameParts[1], theResource);
                        // update artifact with local repo reference
@@ -218,16 +219,16 @@ public class ContentServiceImpl extends AbstractServiceImpl
        public void putDocumentContent(
                String theSolutionId, String theRevisionId, Document theDocument, Resource theResource, ServiceContext theContext)
                                                                                                                                                                                                                                                                                                                                                throws ServiceException {
-               String[] nameParts = splitName(theDocument.getFilename());
+               String[] nameParts = splitName(getName(theDocument));
                String uri = putNexusContent(
                        nexusPrefix(theSolutionId, theRevisionId), nameParts[0], AccessTypeCode.PB.name(), nameParts[1], theResource);
                theDocument.setUri(uri);
        }
 
-       protected Resource getNexusContent(String theUri) throws ServiceException {
+       protected Resource getNexusContent(String thePath) throws ServiceException {
                URI contentUri = null;
                try {
-                       contentUri = new URI(this.nexusConfig.getUrl() + theUri); 
+                       contentUri = new URI(this.nexusConfig.getUrl() + UriUtils.encodePath(thePath, "UTF-8")); 
                        log.info(EELFLoggerDelegate.debugLogger, "Query for {}", contentUri);
                        ResponseEntity<Resource> response = null;
                        RequestEntity<Void> request = RequestEntity
@@ -238,7 +239,7 @@ public class ContentServiceImpl extends AbstractServiceImpl
                        return response.getBody();      
                }
                catch (HttpStatusCodeException x) {
-                       log.error(EELFLoggerDelegate.errorLogger, "Failed to retrieve nexus content from  " + theUri + "(" + contentUri + ")", x);
+                       log.error(EELFLoggerDelegate.errorLogger, "Failed to retrieve nexus content from  " + thePath + "(" + contentUri + ")", x);
                        throw new ServiceException("Failed to retrieve nexus content from " + contentUri, x);
                }
                catch (Throwable t) {
@@ -253,10 +254,10 @@ public class ContentServiceImpl extends AbstractServiceImpl
 
                try {
                        String path = nexusPath(theGroupId, theContentId, theVersion, thePackaging);
-                       URI uri = new URI(this.nexusConfig.getUrl() + path);
+                       URI contentUri = new URI(this.nexusConfig.getUrl() + UriUtils.encodePath(path, "UTF-8"));
                        log.info(EELFLoggerDelegate.debugLogger, "Writing artifact content to nexus at {}", path);
                        RequestEntity<Resource> request = RequestEntity
-                                                                                                                                                                       .put(uri)
+                                                                                                                                                                       .put(contentUri)
                                                                                                                                                                        .contentType(MediaType.APPLICATION_OCTET_STREAM)
                                                                                                                                                                        //.contentLength()
                                                                                                                                                                        .body(theResource);
@@ -314,6 +315,20 @@ public class ContentServiceImpl extends AbstractServiceImpl
                                                                                                                                                new String[] {theName.substring(0,pos), theName.substring(pos+1)};
        }
 
+       private String getName(Artifact theArtifact) {
+               String name  = theArtifact.getFilename();
+               if (name == null)
+                       name = theArtifact.getName();
+               return name;
+       }
+
+       private String getName(Document theDoc) {
+               String name  = theDoc.getFilename();
+               if (name == null)
+                       name = theDoc.getName();
+               return name;
+       }
+
        /**
         * Allows for accurate counting of the amount of data transferred. The docker image info resulting from
         * a 'docker image ls' is slightly different ..
index c20a11a..d7601d0 100644 (file)
@@ -113,8 +113,8 @@ public class PeerSubscriptionTask implements Runnable {
                                }
                        }
 
-                       this.subscription.setProcessed(new Date());
-                       this.peerSubscriptionService.updatePeerSubscription(this.subscription);
+                       //moved this to PeerGateway, not good design but practical ..
+                       //this.peerSubscriptionService.updatePeerSubscription(this.subscription);
                }
                catch (Exception x) {
                        log.error(EELFLoggerDelegate.errorLogger, "Peer task failed for " + peer.getName() + ", " + peer.getApiUrl() + ", " + subscription, x);