001: /*
002: * This file is part of DrFTPD, Distributed FTP Daemon.
003: *
004: * DrFTPD is free software; you can redistribute it and/or modify
005: * it under the terms of the GNU General Public License as published by
006: * the Free Software Foundation; either version 2 of the License, or
007: * (at your option) any later version.
008: *
009: * DrFTPD is distributed in the hope that it will be useful,
010: * but WITHOUT ANY WARRANTY; without even the implied warranty of
011: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
012: * GNU General Public License for more details.
013: *
014: * You should have received a copy of the GNU General Public License
015: * along with DrFTPD; if not, write to the Free Software
016: * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
017: */
018: package org.drftpd.mirroring;
019:
020: import java.io.FileNotFoundException;
021: import java.io.IOException;
022: import java.util.ArrayList;
023: import java.util.Collection;
024: import java.util.Collections;
025: import java.util.HashSet;
026: import java.util.Iterator;
027: import java.util.Properties;
028: import java.util.Set;
029:
030: import net.sf.drftpd.NoAvailableSlaveException;
031: import net.sf.drftpd.ObjectNotFoundException;
032: import net.sf.drftpd.mirroring.Job;
033: import net.sf.drftpd.mirroring.JobManager;
034:
035: import org.apache.log4j.Logger;
036: import org.drftpd.PropertyHelper;
037: import org.drftpd.master.RemoteSlave;
038: import org.drftpd.mirroring.archivetypes.IncompleteDirectoryException;
039: import org.drftpd.mirroring.archivetypes.OfflineSlaveException;
040: import org.drftpd.plugins.Archive;
041: import org.drftpd.remotefile.FileStillTransferringException;
042: import org.drftpd.remotefile.LinkedRemoteFile;
043: import org.drftpd.remotefile.LinkedRemoteFileInterface;
044: import org.drftpd.sections.SectionInterface;
045: import org.drftpd.sections.conf.DatedSection;
046: import org.drftpd.sections.def.SectionManager.Section;
047:
048: /**
049: * @author zubov
050: * @version $Id: ArchiveType.java 1526 2006-10-21 00:16:26Z fr0w $
051: */
052: public abstract class ArchiveType {
053: private static final Logger logger = Logger
054: .getLogger(ArchiveType.class);
055: private long _archiveAfter;
056: private LinkedRemoteFileInterface _directory;
057: protected Archive _parent;
058: protected SectionInterface _section;
059: protected Set<RemoteSlave> _slaveList;
060: protected int _numOfSlaves;
061:
062: /**
063: * Sets _slaveList, _numOfSlaves, _section, _parent, and _archiveAfter
064: * Each implementing ArchiveType still needs to check/validate these figures to their specific needs
065: * After the constructor finishes, _slaveList is guaranteed to be non-null, but can still be empty
066: * @param archive
067: * @param section
068: * @param p
069: */
070: public ArchiveType(Archive archive, SectionInterface section,
071: Properties p) {
072: _parent = archive;
073: _section = section;
074: setProperties(p);
075: }
076:
077: /**
078: * Once the Jobs in the jobList have been sent, this method is called
079: * This is where files are possibly deleted from slaves
080: */
081: public void cleanup(ArrayList<Job> jobList) {
082: for (Job job : jobList) {
083: for (RemoteSlave rslave : new ArrayList<RemoteSlave>(job
084: .getFile().getSlaves())) {
085: if (!getRSlaves().contains(rslave)) {
086: rslave.simpleDelete(job.getFile().getPath());
087: job.getFile().removeSlave(rslave);
088: }
089: }
090: for (RemoteSlave rslave : new ArrayList<RemoteSlave>(job
091: .getFile().getSlaves())) {
092: if (job.getFile().getSlaves().size() > _numOfSlaves) {
093: if (!rslave.isAvailable()) {
094: rslave.simpleDelete(job.getFile().getPath());
095: job.getFile().removeSlave(rslave);
096: }
097: } else {
098: break;
099: }
100: }
101: for (RemoteSlave rslave : new ArrayList<RemoteSlave>(job
102: .getFile().getSlaves())) {
103: if (job.getFile().getSlaves().size() > _numOfSlaves) {
104: rslave.simpleDelete(job.getFile().getPath());
105: job.getFile().removeSlave(rslave);
106: } else {
107: break;
108: }
109: }
110: }
111: }
112:
113: /**
114: * Used to determine a list of slaves dynamically during runtime, only gets called if _slaveList == null
115: * @return
116: */
117: public abstract Set<RemoteSlave> findDestinationSlaves();
118:
119: public final LinkedRemoteFileInterface getDirectory() {
120: return _directory;
121: }
122:
123: /**
124: * Returns the oldest LinkedRemoteFile(directory) that needs to be archived by this type's definition
125: * If no such directory exists, it returns null
126: */
127: public final LinkedRemoteFileInterface getOldestNonArchivedDir() {
128: ArrayList<LinkedRemoteFileInterface> oldDirs = new ArrayList<LinkedRemoteFileInterface>();
129:
130: LinkedRemoteFileInterface baseFile;
131: if (getSection() instanceof DatedSection) {
132: baseFile = getSection().getBaseFile();
133: } else {
134: baseFile = getSection().getFile();
135: }
136:
137: for (Iterator iter = baseFile.getFiles().iterator(); iter
138: .hasNext();) {
139: LinkedRemoteFileInterface lrf = (LinkedRemoteFileInterface) iter
140: .next();
141: if (!lrf.isDirectory()) {
142: continue;
143: }
144: try {
145: _parent.checkPathForArchiveStatus(lrf.getPath());
146: } catch (DuplicateArchiveException e1) {
147: continue;
148: }
149:
150: try {
151: if ((System.currentTimeMillis() - lrf.lastModified()) > getArchiveAfter()) {
152: if (!isArchivedDir(lrf)) {
153: oldDirs.add(lrf);
154: }
155: }
156: } catch (IncompleteDirectoryException e) {
157: continue;
158: } catch (OfflineSlaveException e) {
159: continue;
160: }
161: }
162:
163: LinkedRemoteFileInterface oldestDir = null;
164:
165: for (Iterator iter = oldDirs.iterator(); iter.hasNext();) {
166: LinkedRemoteFileInterface temp = (LinkedRemoteFileInterface) iter
167: .next();
168:
169: if (oldestDir == null) {
170: oldestDir = temp;
171:
172: continue;
173: }
174:
175: if (oldestDir.lastModified() > temp.lastModified()) {
176: oldestDir = temp;
177: }
178: }
179:
180: if (oldestDir != null) {
181: logger.debug(getClass().toString()
182: + " - Returning the oldest directory " + oldestDir);
183: } else {
184: logger.debug(getClass().toString()
185: + " - Returning a null directory");
186: }
187:
188: return oldestDir;
189: }
190:
191: /**
192: * if the directory is archived by this type's definition, this method returns true
193: */
194: protected abstract boolean isArchivedDir(
195: LinkedRemoteFileInterface lrf)
196: throws IncompleteDirectoryException, OfflineSlaveException;
197:
198: /**
199: * Returns unmodifiable Set<RemoteSlave>.
200: */
201: public final Set<RemoteSlave> getRSlaves() {
202: return _slaveList == null ? null : Collections
203: .unmodifiableSet(_slaveList);
204: }
205:
206: /**
207: * Adds relevant Jobs to the JobManager and returns an ArrayList of those Job's
208: */
209: public ArrayList<Job> send() {
210: ArrayList<Job> jobs = recursiveSend(getDirectory());
211: JobManager jm = _parent.getGlobalContext().getJobManager();
212: jm.addJobsToQueue(jobs);
213: return jobs;
214: }
215:
216: protected ArrayList<Job> recursiveSend(LinkedRemoteFileInterface lrf) {
217: ArrayList<Job> jobQueue = new ArrayList<Job>();
218:
219: for (Iterator iter = lrf.getFiles().iterator(); iter.hasNext();) {
220: LinkedRemoteFileInterface src = (LinkedRemoteFileInterface) iter
221: .next();
222:
223: if (src.isFile()) {
224: logger.info("Adding " + src.getPath()
225: + " to the job queue");
226:
227: Job job = new Job(src, getRSlaves(), 3, _numOfSlaves);
228: jobQueue.add(job);
229: } else {
230: jobQueue.addAll(recursiveSend(src));
231: }
232: }
233:
234: return jobQueue;
235: }
236:
237: protected static final boolean isArchivedToXSlaves(
238: LinkedRemoteFileInterface lrf, int x)
239: throws IncompleteDirectoryException, OfflineSlaveException {
240: HashSet<RemoteSlave> slaveSet = null;
241:
242: if (lrf.getFiles().isEmpty()) {
243: return true;
244: }
245:
246: try {
247: try {
248: if (!lrf.lookupSFVFile().getStatus().isFinished()) {
249: logger.debug(lrf.getPath() + " is not complete");
250: throw new IncompleteDirectoryException(lrf
251: .getPath()
252: + " is not complete");
253: }
254: } catch (FileStillTransferringException e) {
255: logger.debug(lrf.getPath()
256: + " is still transferrring it's SFV");
257: throw new IncompleteDirectoryException(lrf.getPath()
258: + " is still transferrring it's SFV");
259: }
260: } catch (FileNotFoundException e) {
261: } catch (IOException e) {
262: } catch (NoAvailableSlaveException e) {
263: throw new OfflineSlaveException("SFV is offline", e);
264: }
265:
266: for (Iterator iter = lrf.getFiles().iterator(); iter.hasNext();) {
267: LinkedRemoteFileInterface file = (LinkedRemoteFileInterface) iter
268: .next();
269:
270: if (file.isDirectory()) {
271: if (!isArchivedToXSlaves(file, x)) {
272: return false;
273: }
274: } else { // file.isFile()
275: if (!file.isAvailable())
276: throw new OfflineSlaveException(file.getPath()
277: + " is offline");
278: Collection<RemoteSlave> availableSlaves = file
279: .getSlaves();
280:
281: if (slaveSet == null) {
282: slaveSet = new HashSet<RemoteSlave>(availableSlaves);
283: } else {
284: if (!(slaveSet.containsAll(availableSlaves) && availableSlaves
285: .containsAll(slaveSet))) {
286: return false;
287: }
288: }
289: }
290: }
291:
292: if (slaveSet == null) { // no files found in directory
293: return true;
294: }
295:
296: for (Iterator iter = slaveSet.iterator(); iter.hasNext();) {
297: RemoteSlave rslave = (RemoteSlave) iter.next();
298:
299: if (!rslave.isAvailable()) {
300: throw new OfflineSlaveException(rslave.getName()
301: + " is offline");
302: }
303: }
304:
305: return (slaveSet.size() == x);
306: }
307:
308: public final boolean isBusy() {
309: return (getDirectory() != null);
310: }
311:
312: protected final long getArchiveAfter() {
313: return _archiveAfter;
314: }
315:
316: public final SectionInterface getSection() {
317: return _section;
318: }
319:
320: /**
321: * Sets standard properties for this ArchiveType
322: */
323: private void setProperties(Properties properties) {
324: try {
325: _archiveAfter = 60000 * Long.parseLong(PropertyHelper
326: .getProperty(properties, getSection().getName()
327: + ".archiveAfter"));
328: } catch (NullPointerException e) {
329: _archiveAfter = 0;
330: }
331: _numOfSlaves = Integer.parseInt(properties.getProperty(
332: getSection().getName() + ".numOfSlaves", "0"));
333:
334: HashSet<RemoteSlave> destSlaves = new HashSet<RemoteSlave>();
335:
336: for (int i = 1;; i++) {
337: String slavename = null;
338:
339: try {
340: slavename = PropertyHelper.getProperty(properties,
341: getSection().getName() + ".slavename." + i);
342: } catch (NullPointerException e) {
343: break; // done
344: }
345:
346: try {
347: RemoteSlave rslave = _parent.getGlobalContext()
348: .getSlaveManager().getRemoteSlave(slavename);
349: destSlaves.add(rslave);
350: } catch (ObjectNotFoundException e) {
351: logger.error("Unable to get slave " + slavename
352: + " from the SlaveManager");
353: }
354: }
355: _slaveList = destSlaves;
356: }
357:
358: public final void setDirectory(LinkedRemoteFileInterface lrf) {
359: _directory = lrf;
360: }
361:
362: public final void setRSlaves(Set<RemoteSlave> slaveList) {
363: _slaveList = slaveList;
364: }
365:
366: public final void waitForSendOfFiles(ArrayList<Job> jobQueue) {
367: while (true) {
368: if (_directory.isDeleted()) {
369: // all files will be deleted too, no need to removejobs, JobManager will do that
370: return;
371: }
372: for (Iterator iter = jobQueue.iterator(); iter.hasNext();) {
373: Job job = (Job) iter.next();
374:
375: if (job.isDone()) {
376: logger.debug("Job " + job + " is done being sent");
377: iter.remove();
378: }
379: }
380:
381: try {
382: Thread.sleep(10000);
383: } catch (InterruptedException e) {
384: }
385:
386: if (jobQueue.isEmpty()) {
387: break;
388: }
389: }
390: }
391:
392: private void printJobsInQueue(ArrayList<Job> jobList) {
393: for (Job j : jobList) {
394: logger.debug(j);
395: }
396: }
397:
398: public abstract String toString();
399:
400: protected String outputSlaves(Collection slaveList) {
401: String toReturn = new String();
402:
403: for (Iterator iter = slaveList.iterator(); iter.hasNext();) {
404: RemoteSlave rslave = (RemoteSlave) iter.next();
405: toReturn = toReturn + rslave.getName();
406:
407: if (iter.hasNext()) {
408: toReturn = toReturn + ",";
409: } else {
410: return toReturn;
411: }
412: }
413:
414: return "Empty";
415: }
416:
417: public Set<RemoteSlave> getOffOfSlaves(Properties props) {
418: Set<RemoteSlave> offOfSlaves = new HashSet<RemoteSlave>();
419:
420: for (int i = 1;; i++) {
421: String slavename = null;
422:
423: try {
424: slavename = PropertyHelper.getProperty(props,
425: getSection().getName() + ".offOfSlave." + i);
426: } catch (NullPointerException e) {
427: break; // done
428: }
429:
430: try {
431: RemoteSlave rslave = _parent.getGlobalContext()
432: .getSlaveManager().getRemoteSlave(slavename);
433: if (!_slaveList.contains(rslave)) {
434: offOfSlaves.add(rslave);
435: }
436: } catch (ObjectNotFoundException e) {
437: logger.error("Unable to get slave " + slavename
438: + " from the SlaveManager");
439: }
440: }
441: return offOfSlaves;
442:
443: }
444: }
|