001: /*
002: * Licensed to the Apache Software Foundation (ASF) under one or more
003: * contributor license agreements. See the NOTICE file distributed with
004: * this work for additional information regarding copyright ownership.
005: * The ASF licenses this file to You under the Apache License, Version 2.0
006: * (the "License"); you may not use this file except in compliance with
007: * the License. You may obtain a copy of the License at
008: *
009: * http://www.apache.org/licenses/LICENSE-2.0
010: *
011: * Unless required by applicable law or agreed to in writing, software
012: * distributed under the License is distributed on an "AS IS" BASIS,
013: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014: * See the License for the specific language governing permissions and
015: * limitations under the License.
016: *
017: */
018: package org.apache.ivy.core.retrieve;
019:
020: import java.io.File;
021: import java.io.IOException;
022: import java.net.URL;
023: import java.text.ParseException;
024: import java.util.ArrayList;
025: import java.util.Arrays;
026: import java.util.Collection;
027: import java.util.Collections;
028: import java.util.Comparator;
029: import java.util.HashMap;
030: import java.util.HashSet;
031: import java.util.Iterator;
032: import java.util.List;
033: import java.util.Map;
034: import java.util.Set;
035:
036: import org.apache.ivy.core.IvyContext;
037: import org.apache.ivy.core.IvyPatternHelper;
038: import org.apache.ivy.core.LogOptions;
039: import org.apache.ivy.core.cache.ResolutionCacheManager;
040: import org.apache.ivy.core.event.EventManager;
041: import org.apache.ivy.core.event.retrieve.EndRetrieveEvent;
042: import org.apache.ivy.core.event.retrieve.StartRetrieveEvent;
043: import org.apache.ivy.core.module.descriptor.Artifact;
044: import org.apache.ivy.core.module.descriptor.ModuleDescriptor;
045: import org.apache.ivy.core.module.id.ModuleId;
046: import org.apache.ivy.core.module.id.ModuleRevisionId;
047: import org.apache.ivy.core.report.ArtifactDownloadReport;
048: import org.apache.ivy.core.resolve.ResolveOptions;
049: import org.apache.ivy.plugins.parser.ModuleDescriptorParser;
050: import org.apache.ivy.plugins.parser.ModuleDescriptorParserRegistry;
051: import org.apache.ivy.plugins.report.XmlReportParser;
052: import org.apache.ivy.plugins.repository.url.URLResource;
053: import org.apache.ivy.util.FileUtil;
054: import org.apache.ivy.util.Message;
055:
056: public class RetrieveEngine {
057: private static final int KILO = 1024;
058:
059: private RetrieveEngineSettings settings;
060:
061: private EventManager eventManager;
062:
063: public RetrieveEngine(RetrieveEngineSettings settings,
064: EventManager eventManager) {
065: this .settings = settings;
066: this .eventManager = eventManager;
067: }
068:
069: /**
070: * example of destFilePattern : - lib/[organisation]/[module]/[artifact]-[revision].[type] -
071: * lib/[artifact].[type] : flatten with no revision moduleId is used with confs and
072: * localCacheDirectory to determine an ivy report file, used as input for the copy If such a
073: * file does not exist for any conf (resolve has not been called before ?) then an
074: * IllegalStateException is thrown and nothing is copied.
075: */
076: public int retrieve(ModuleRevisionId mrid, String destFilePattern,
077: RetrieveOptions options) throws IOException {
078: ModuleId moduleId = mrid.getModuleId();
079: if (LogOptions.LOG_DEFAULT.equals(options.getLog())) {
080: Message.info(":: retrieving :: " + moduleId
081: + (options.isSync() ? " [sync]" : ""));
082: } else {
083: Message.verbose(":: retrieving :: " + moduleId
084: + (options.isSync() ? " [sync]" : ""));
085: }
086: Message
087: .verbose("\tcheckUpToDate="
088: + settings.isCheckUpToDate());
089: long start = System.currentTimeMillis();
090:
091: destFilePattern = IvyPatternHelper.substituteVariables(
092: destFilePattern, settings.getVariables());
093: String destIvyPattern = IvyPatternHelper.substituteVariables(
094: options.getDestIvyPattern(), settings.getVariables());
095:
096: String[] confs = getConfs(mrid, options);
097: if (LogOptions.LOG_DEFAULT.equals(options.getLog())) {
098: Message.info("\tconfs: " + Arrays.asList(confs));
099: } else {
100: Message.verbose("\tconfs: " + Arrays.asList(confs));
101: }
102: if (this .eventManager != null) {
103: this .eventManager.fireIvyEvent(new StartRetrieveEvent(mrid,
104: confs, options));
105: }
106:
107: try {
108: Map artifactsToCopy = determineArtifactsToCopy(mrid,
109: destFilePattern, options);
110: File fileRetrieveRoot = new File(IvyPatternHelper
111: .getTokenRoot(destFilePattern));
112: File ivyRetrieveRoot = destIvyPattern == null ? null
113: : new File(IvyPatternHelper
114: .getTokenRoot(destIvyPattern));
115: Collection targetArtifactsStructure = new HashSet(); // Set(File) set of all paths
116: // which should be present at
117: // then end of retrieve (useful
118: // for sync)
119: Collection targetIvysStructure = new HashSet(); // same for ivy files
120:
121: // do retrieve
122: int targetsCopied = 0;
123: int targetsUpToDate = 0;
124: long totalCopiedSize = 0;
125: for (Iterator iter = artifactsToCopy.keySet().iterator(); iter
126: .hasNext();) {
127: ArtifactDownloadReport artifact = (ArtifactDownloadReport) iter
128: .next();
129: File archive = artifact.getLocalFile();
130: if (archive == null) {
131: Message.verbose("\tno local file available for "
132: + artifact + ": skipping");
133: continue;
134: }
135: Set dest = (Set) artifactsToCopy.get(artifact);
136: Message.verbose("\tretrieving " + archive);
137: for (Iterator it2 = dest.iterator(); it2.hasNext();) {
138: IvyContext.getContext().checkInterrupted();
139: File destFile = new File((String) it2.next());
140: if (!settings.isCheckUpToDate()
141: || !upToDate(archive, destFile)) {
142: Message.verbose("\t\tto " + destFile);
143: if (options.isMakeSymlinks()) {
144: FileUtil.symlink(archive, destFile, null,
145: true);
146: } else {
147: FileUtil
148: .copy(archive, destFile, null, true);
149: }
150: totalCopiedSize += destFile.length();
151: targetsCopied++;
152: } else {
153: Message.verbose("\t\tto " + destFile
154: + " [NOT REQUIRED]");
155: targetsUpToDate++;
156: }
157: if ("ivy".equals(artifact.getType())) {
158: targetIvysStructure
159: .addAll(FileUtil.getPathFiles(
160: ivyRetrieveRoot, destFile));
161: } else {
162: targetArtifactsStructure.addAll(FileUtil
163: .getPathFiles(fileRetrieveRoot,
164: destFile));
165: }
166: }
167: }
168:
169: if (options.isSync()) {
170: Message.verbose("\tsyncing...");
171: Collection existingArtifacts = FileUtil
172: .listAll(fileRetrieveRoot);
173: Collection existingIvys = ivyRetrieveRoot == null ? null
174: : FileUtil.listAll(ivyRetrieveRoot);
175:
176: if (fileRetrieveRoot.equals(ivyRetrieveRoot)) {
177: Collection target = targetArtifactsStructure;
178: target.addAll(targetIvysStructure);
179: Collection existing = existingArtifacts;
180: existing.addAll(existingIvys);
181: sync(target, existing);
182: } else {
183: sync(targetArtifactsStructure, existingArtifacts);
184: if (existingIvys != null) {
185: sync(targetIvysStructure, existingIvys);
186: }
187: }
188: }
189: long elapsedTime = System.currentTimeMillis() - start;
190: String msg = "\t"
191: + targetsCopied
192: + " artifacts copied"
193: + (settings.isCheckUpToDate() ? (", "
194: + targetsUpToDate + " already retrieved")
195: : "") + " (" + (totalCopiedSize / KILO)
196: + "kB/" + elapsedTime + "ms)";
197: if (LogOptions.LOG_DEFAULT.equals(options.getLog())) {
198: Message.info(msg);
199: } else {
200: Message.verbose(msg);
201: }
202: Message
203: .verbose("\tretrieve done (" + (elapsedTime)
204: + "ms)");
205: if (this .eventManager != null) {
206: this .eventManager.fireIvyEvent(new EndRetrieveEvent(
207: mrid, confs, elapsedTime, targetsCopied,
208: targetsUpToDate, totalCopiedSize, options));
209: }
210:
211: return targetsCopied;
212: } catch (Exception ex) {
213: throw new RuntimeException("problem during retrieve of "
214: + moduleId + ": " + ex, ex);
215: }
216: }
217:
218: private String[] getConfs(ModuleRevisionId mrid,
219: RetrieveOptions options) throws IOException {
220: String[] confs = options.getConfs();
221: if (confs == null
222: || (confs.length == 1 && "*".equals(confs[0]))) {
223: try {
224: File ivyFile = getCache().getResolvedIvyFileInCache(
225: mrid);
226: Message
227: .verbose("no explicit confs given for retrieve, using ivy file: "
228: + ivyFile);
229: URL ivySource = ivyFile.toURL();
230: URLResource res = new URLResource(ivySource);
231: ModuleDescriptorParser parser = ModuleDescriptorParserRegistry
232: .getInstance().getParser(res);
233: Message.debug("using " + parser + " to parse "
234: + ivyFile);
235: ModuleDescriptor md = parser.parseDescriptor(settings,
236: ivySource, false);
237: confs = md.getConfigurationsNames();
238: options.setConfs(confs);
239: } catch (IOException e) {
240: throw e;
241: } catch (Exception e) {
242: IOException ioex = new IOException(e.getMessage());
243: ioex.initCause(e);
244: throw ioex;
245: }
246: }
247: return confs;
248: }
249:
250: private ResolutionCacheManager getCache() {
251: return settings.getResolutionCacheManager();
252: }
253:
254: private void sync(Collection target, Collection existing) {
255: Collection toRemove = new HashSet();
256: for (Iterator iter = existing.iterator(); iter.hasNext();) {
257: File file = (File) iter.next();
258: toRemove.add(file.getAbsoluteFile());
259: }
260: for (Iterator iter = target.iterator(); iter.hasNext();) {
261: File file = (File) iter.next();
262: toRemove.remove(file.getAbsoluteFile());
263: }
264: for (Iterator iter = toRemove.iterator(); iter.hasNext();) {
265: File file = (File) iter.next();
266: if (file.exists()) {
267: Message.verbose("\t\tdeleting " + file);
268: FileUtil.forceDelete(file);
269: }
270: }
271: }
272:
273: public Map determineArtifactsToCopy(ModuleRevisionId mrid,
274: String destFilePattern, RetrieveOptions options)
275: throws ParseException, IOException {
276: ModuleId moduleId = mrid.getModuleId();
277:
278: if (options.getResolveId() == null) {
279: options.setResolveId(ResolveOptions
280: .getDefaultResolveId(moduleId));
281: }
282:
283: ResolutionCacheManager cacheManager = getCache();
284: String[] confs = getConfs(mrid, options);
285: String destIvyPattern = IvyPatternHelper.substituteVariables(
286: options.getDestIvyPattern(), settings.getVariables());
287:
288: // find what we must retrieve where
289:
290: // ArtifactDownloadReport source -> Set (String copyDestAbsolutePath)
291: final Map artifactsToCopy = new HashMap();
292: // String copyDestAbsolutePath -> Set (ArtifactDownloadReport source)
293: final Map conflictsMap = new HashMap();
294: // String copyDestAbsolutePath -> Set (String conf)
295: final Map conflictsConfMap = new HashMap();
296:
297: XmlReportParser parser = new XmlReportParser();
298: for (int i = 0; i < confs.length; i++) {
299: final String conf = confs[i];
300:
301: File report = cacheManager
302: .getConfigurationResolveReportInCache(options
303: .getResolveId(), conf);
304: parser.parse(report);
305:
306: Collection artifacts = new ArrayList(Arrays.asList(parser
307: .getArtifactReports()));
308: if (destIvyPattern != null) {
309: ModuleRevisionId[] mrids = parser
310: .getRealDependencyRevisionIds();
311: for (int j = 0; j < mrids.length; j++) {
312: artifacts.add(parser
313: .getMetadataArtifactReport(mrids[j]));
314: }
315: }
316: for (Iterator iter = artifacts.iterator(); iter.hasNext();) {
317: ArtifactDownloadReport artifact = (ArtifactDownloadReport) iter
318: .next();
319: String destPattern = "ivy".equals(artifact.getType()) ? destIvyPattern
320: : destFilePattern;
321:
322: if (!"ivy".equals(artifact.getType())
323: && !options.getArtifactFilter().accept(
324: artifact.getArtifact())) {
325: continue; // skip this artifact, the filter didn't accept it!
326: }
327:
328: String destFileName = IvyPatternHelper.substitute(
329: destPattern, artifact.getArtifact(), conf);
330:
331: Set dest = (Set) artifactsToCopy.get(artifact);
332: if (dest == null) {
333: dest = new HashSet();
334: artifactsToCopy.put(artifact, dest);
335: }
336: String copyDest = new File(destFileName)
337: .getAbsolutePath();
338: dest.add(copyDest);
339:
340: Set conflicts = (Set) conflictsMap.get(copyDest);
341: Set conflictsConf = (Set) conflictsConfMap
342: .get(copyDest);
343: if (conflicts == null) {
344: conflicts = new HashSet();
345: conflictsMap.put(copyDest, conflicts);
346: }
347: if (conflictsConf == null) {
348: conflictsConf = new HashSet();
349: conflictsConfMap.put(copyDest, conflictsConf);
350: }
351: conflicts.add(artifact);
352: conflictsConf.add(conf);
353: }
354: }
355:
356: // resolve conflicts if any
357: for (Iterator iter = conflictsMap.keySet().iterator(); iter
358: .hasNext();) {
359: String copyDest = (String) iter.next();
360: Set artifacts = (Set) conflictsMap.get(copyDest);
361: Set conflictsConfs = (Set) conflictsConfMap.get(copyDest);
362: if (artifacts.size() > 1) {
363: List artifactsList = new ArrayList(artifacts);
364: // conflicts battle is resolved by a sort using a conflict resolving policy
365: // comparator
366: // which consider as greater a winning artifact
367: Collections.sort(artifactsList,
368: getConflictResolvingPolicy());
369: // after the sort, the winning artifact is the greatest one, i.e. the last one
370: Message.info("\tconflict on "
371: + copyDest
372: + " in "
373: + conflictsConfs
374: + ": "
375: + ((ArtifactDownloadReport) artifactsList
376: .get(artifactsList.size() - 1))
377: .getArtifact().getModuleRevisionId()
378: .getRevision() + " won");
379:
380: // we now iterate over the list beginning with the artifact preceding the winner,
381: // and going backward to the least artifact
382: for (int i = artifactsList.size() - 2; i >= 0; i--) {
383: ArtifactDownloadReport looser = (ArtifactDownloadReport) artifactsList
384: .get(i);
385: Message
386: .verbose("\t\tremoving conflict looser artifact: "
387: + looser.getArtifact());
388: // for each loser, we remove the pair (loser - copyDest) in the artifactsToCopy
389: // map
390: Set dest = (Set) artifactsToCopy.get(looser);
391: dest.remove(copyDest);
392: if (dest.isEmpty()) {
393: artifactsToCopy.remove(looser);
394: }
395: }
396: }
397: }
398: return artifactsToCopy;
399: }
400:
401: private boolean upToDate(File source, File target) {
402: if (!target.exists()) {
403: return false;
404: }
405: return source.lastModified() <= target.lastModified();
406: }
407:
408: /**
409: * The returned comparator should consider greater the artifact which gains the conflict battle.
410: * This is used only during retrieve... prefer resolve conflict manager to resolve conflicts.
411: *
412: * @return
413: */
414: private Comparator getConflictResolvingPolicy() {
415: return new Comparator() {
416: // younger conflict resolving policy
417: public int compare(Object o1, Object o2) {
418: Artifact a1 = ((ArtifactDownloadReport) o1)
419: .getArtifact();
420: Artifact a2 = ((ArtifactDownloadReport) o2)
421: .getArtifact();
422: if (a1.getPublicationDate().after(
423: a2.getPublicationDate())) {
424: // a1 is after a2 <=> a1 is younger than a2 <=> a1 wins the conflict battle
425: return +1;
426: } else if (a1.getPublicationDate().before(
427: a2.getPublicationDate())) {
428: // a1 is before a2 <=> a2 is younger than a1 <=> a2 wins the conflict battle
429: return -1;
430: } else {
431: return 0;
432: }
433: }
434: };
435: }
436:
437: }
|