OpenSim
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Events Macros
InventoryArchiveReadRequest.cs
Go to the documentation of this file.
1 /*
2  * Copyright (c) Contributors, http://opensimulator.org/
3  * See CONTRIBUTORS.TXT for a full list of copyright holders.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  * * Redistributions of source code must retain the above copyright
8  * notice, this list of conditions and the following disclaimer.
9  * * Redistributions in binary form must reproduce the above copyright
10  * notice, this list of conditions and the following disclaimer in the
11  * documentation and/or other materials provided with the distribution.
12  * * Neither the name of the OpenSimulator Project nor the
13  * names of its contributors may be used to endorse or promote products
14  * derived from this software without specific prior written permission.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19  * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 using System;
29 using System.Collections.Generic;
30 using System.IO;
31 using System.IO.Compression;
32 using System.Reflection;
33 using System.Threading;
34 using System.Text;
35 using System.Xml;
36 using System.Xml.Linq;
37 using log4net;
38 using OpenMetaverse;
39 using OpenSim.Framework;
40 using OpenSim.Framework.Serialization;
41 using OpenSim.Framework.Serialization.External;
42 using OpenSim.Region.CoreModules.World.Archiver;
43 using OpenSim.Region.Framework.Scenes;
44 using OpenSim.Region.Framework.Scenes.Serialization;
45 using OpenSim.Region.Framework.Interfaces;
46 using OpenSim.Services.Interfaces;
47 
48 namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
49 {
51  {
52  private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
53 
58  public static int MAX_MAJOR_VERSION = 1;
59 
61 
62  private UserAccount m_userInfo;
63  private string m_invPath;
64 
68  protected UUID m_id;
69 
73  protected bool m_merge;
74 
78 
79  private InventoryArchiverModule m_module;
80 
84  private Stream m_loadStream;
85 
89  public bool ControlFileLoaded { get; private set; }
90 
95  public bool EnforceControlFileCheck { get; private set; }
96 
97  protected bool m_assetsLoaded;
98  protected bool m_inventoryNodesLoaded;
99 
101  protected int m_failedAssetRestores;
102  protected int m_successfulItemRestores;
103 
108 
112  protected HashSet<InventoryNodeBase> m_loadedNodes = new HashSet<InventoryNodeBase>();
113 
118  Dictionary <string, InventoryFolderBase> m_resolvedFolders = new Dictionary<string, InventoryFolderBase>();
119 
124  protected Dictionary<UUID, UUID> m_creatorIdForAssetId = new Dictionary<UUID, UUID>();
125 
127  IInventoryService inv, IAssetService assets, IUserAccountService uacc, UserAccount userInfo, string invPath, string loadPath, bool merge)
128  : this(UUID.Zero, null,
129  inv,
130  assets,
131  uacc,
132  userInfo,
133  invPath,
134  loadPath,
135  merge)
136  {
137  }
138 
140  UUID id, InventoryArchiverModule module, IInventoryService inv, IAssetService assets, IUserAccountService uacc, UserAccount userInfo, string invPath, string loadPath, bool merge)
141  : this(
142  id,
143  module,
144  inv,
145  assets,
146  uacc,
147  userInfo,
148  invPath,
149  new GZipStream(ArchiveHelpers.GetStream(loadPath), CompressionMode.Decompress),
150  merge)
151  {
152  }
153 
155  UUID id, InventoryArchiverModule module, IInventoryService inv, IAssetService assets, IUserAccountService uacc, UserAccount userInfo, string invPath, Stream loadStream, bool merge)
156  {
157  m_id = id;
158  m_InventoryService = inv;
159  m_AssetService = assets;
160  m_UserAccountService = uacc;
161  m_merge = merge;
162  m_userInfo = userInfo;
163  m_invPath = invPath;
164  m_loadStream = loadStream;
165  m_module = module;
166 
167  // FIXME: Do not perform this check since older versions of OpenSim do save the control file after other things
168  // (I thought they weren't). We will need to bump the version number and perform this check on all
169  // subsequent IAR versions only
170  ControlFileLoaded = true;
171  }
172 
184  public HashSet<InventoryNodeBase> Execute()
185  {
186  try
187  {
188  Exception reportedException = null;
189 
190  string filePath = "ERROR";
191 
192  List<InventoryFolderBase> folderCandidates
193  = InventoryArchiveUtils.FindFoldersByPath(
194  m_InventoryService, m_userInfo.PrincipalID, m_invPath);
195 
196  if (folderCandidates.Count == 0)
197  {
198  // Possibly provide an option later on to automatically create this folder if it does not exist
199  m_log.ErrorFormat("[INVENTORY ARCHIVER]: Inventory path {0} does not exist", m_invPath);
200 
201  return m_loadedNodes;
202  }
203 
204  m_rootDestinationFolder = folderCandidates[0];
205  archive = new TarArchiveReader(m_loadStream);
206  byte[] data;
207  TarArchiveReader.TarEntryType entryType;
208 
209  while ((data = archive.ReadEntry(out filePath, out entryType)) != null)
210  {
211  if (filePath == ArchiveConstants.CONTROL_FILE_PATH)
212  {
213  LoadControlFile(filePath, data);
214  }
215  else if (filePath.StartsWith(ArchiveConstants.ASSETS_PATH))
216  {
217  LoadAssetFile(filePath, data);
218  }
219  else if (filePath.StartsWith(ArchiveConstants.INVENTORY_PATH))
220  {
221  LoadInventoryFile(filePath, entryType, data);
222  }
223  }
224 
225  archive.Close();
226 
227  m_log.DebugFormat(
228  "[INVENTORY ARCHIVER]: Successfully loaded {0} assets with {1} failures",
229  m_successfulAssetRestores, m_failedAssetRestores);
230 
231  //Alicia: When this is called by LibraryModule or Tests, m_module will be null as event is not required
232  if(m_module != null)
233  m_module.TriggerInventoryArchiveLoaded(m_id, true, m_userInfo, m_invPath, m_loadStream, reportedException, m_successfulItemRestores);
234 
235  return m_loadedNodes;
236  }
237  catch(Exception Ex)
238  {
239  // Trigger saved event with failed result and exception data
240  if (m_module != null)
241  m_module.TriggerInventoryArchiveLoaded(m_id, false, m_userInfo, m_invPath, m_loadStream, Ex, 0);
242 
243  return m_loadedNodes;
244  }
245  finally
246  {
247  m_loadStream.Close();
248  }
249  }
250 
251  public void Close()
252  {
253  if (m_loadStream != null)
254  m_loadStream.Close();
255  }
256 
271  string iarPath,
272  InventoryFolderBase rootDestFolder,
273  Dictionary <string, InventoryFolderBase> resolvedFolders,
274  HashSet<InventoryNodeBase> loadedNodes)
275  {
276  string iarPathExisting = iarPath;
277 
278 // m_log.DebugFormat(
279 // "[INVENTORY ARCHIVER]: Loading folder {0} {1}", rootDestFolder.Name, rootDestFolder.ID);
280 
281  InventoryFolderBase destFolder
282  = ResolveDestinationFolder(rootDestFolder, ref iarPathExisting, resolvedFolders);
283 
284 // m_log.DebugFormat(
285 // "[INVENTORY ARCHIVER]: originalArchivePath [{0}], section already loaded [{1}]",
286 // iarPath, iarPathExisting);
287 
288  string iarPathToCreate = iarPath.Substring(iarPathExisting.Length);
289  CreateFoldersForPath(destFolder, iarPathExisting, iarPathToCreate, resolvedFolders, loadedNodes);
290 
291  return destFolder;
292  }
293 
315  InventoryFolderBase rootDestFolder,
316  ref string archivePath,
317  Dictionary <string, InventoryFolderBase> resolvedFolders)
318  {
319 // string originalArchivePath = archivePath;
320 
321  while (archivePath.Length > 0)
322  {
323 // m_log.DebugFormat("[INVENTORY ARCHIVER]: Trying to resolve destination folder {0}", archivePath);
324 
325  if (resolvedFolders.ContainsKey(archivePath))
326  {
327 // m_log.DebugFormat(
328 // "[INVENTORY ARCHIVER]: Found previously created folder from archive path {0}", archivePath);
329  return resolvedFolders[archivePath];
330  }
331  else
332  {
333  if (m_merge)
334  {
335  // TODO: Using m_invPath is totally wrong - what we need to do is strip the uuid from the
336  // iar name and try to find that instead.
337  string plainPath = ArchiveConstants.ExtractPlainPathFromIarPath(archivePath);
338  List<InventoryFolderBase> folderCandidates
339  = InventoryArchiveUtils.FindFoldersByPath(
340  m_InventoryService, m_userInfo.PrincipalID, plainPath);
341 
342  if (folderCandidates.Count != 0)
343  {
344  InventoryFolderBase destFolder = folderCandidates[0];
345  resolvedFolders[archivePath] = destFolder;
346  return destFolder;
347  }
348  }
349 
350  // Don't include the last slash so find the penultimate one
351  int penultimateSlashIndex = archivePath.LastIndexOf("/", archivePath.Length - 2);
352 
353  if (penultimateSlashIndex >= 0)
354  {
355  // Remove the last section of path so that we can see if we've already resolved the parent
356  archivePath = archivePath.Remove(penultimateSlashIndex + 1);
357  }
358  else
359  {
360 // m_log.DebugFormat(
361 // "[INVENTORY ARCHIVER]: Found no previously created folder for archive path {0}",
362 // originalArchivePath);
363  archivePath = string.Empty;
364  return rootDestFolder;
365  }
366  }
367  }
368 
369  return rootDestFolder;
370  }
371 
390  protected void CreateFoldersForPath(
391  InventoryFolderBase destFolder,
392  string iarPathExisting,
393  string iarPathToReplicate,
394  Dictionary <string, InventoryFolderBase> resolvedFolders,
395  HashSet<InventoryNodeBase> loadedNodes)
396  {
397  string[] rawDirsToCreate = iarPathToReplicate.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
398 
399  for (int i = 0; i < rawDirsToCreate.Length; i++)
400  {
401 // m_log.DebugFormat("[INVENTORY ARCHIVER]: Creating folder {0} from IAR", rawDirsToCreate[i]);
402 
403  if (!rawDirsToCreate[i].Contains(ArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR))
404  continue;
405 
406  int identicalNameIdentifierIndex
407  = rawDirsToCreate[i].LastIndexOf(
408  ArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR);
409 
410  string newFolderName = rawDirsToCreate[i].Remove(identicalNameIdentifierIndex);
411 
412  newFolderName = InventoryArchiveUtils.UnescapeArchivePath(newFolderName);
413  UUID newFolderId = UUID.Random();
414 
415  destFolder
416  = new InventoryFolderBase(
417  newFolderId, newFolderName, m_userInfo.PrincipalID,
418  (short)FolderType.None, destFolder.ID, 1);
419  m_InventoryService.AddFolder(destFolder);
420 
421  // Record that we have now created this folder
422  iarPathExisting += rawDirsToCreate[i] + "/";
423  m_log.DebugFormat("[INVENTORY ARCHIVER]: Created folder {0} from IAR", iarPathExisting);
424  resolvedFolders[iarPathExisting] = destFolder;
425 
426  if (0 == i)
427  loadedNodes.Add(destFolder);
428  }
429  }
430 
438  protected InventoryItemBase LoadItem(byte[] data, InventoryFolderBase loadFolder)
439  {
440  InventoryItemBase item = UserInventoryItemSerializer.Deserialize(data);
441 
442  // Don't use the item ID that's in the file
443  item.ID = UUID.Random();
444 
445  UUID ospResolvedId = OspResolver.ResolveOspa(item.CreatorId, m_UserAccountService);
446  if (UUID.Zero != ospResolvedId) // The user exists in this grid
447  {
448 // m_log.DebugFormat("[INVENTORY ARCHIVER]: Found creator {0} via OSPA resolution", ospResolvedId);
449 
450 // item.CreatorIdAsUuid = ospResolvedId;
451 
452  // Don't preserve the OSPA in the creator id (which actually gets persisted to the
453  // database). Instead, replace with the UUID that we found.
454  item.CreatorId = ospResolvedId.ToString();
455  item.CreatorData = string.Empty;
456  }
457  else if (string.IsNullOrEmpty(item.CreatorData))
458  {
459  item.CreatorId = m_userInfo.PrincipalID.ToString();
460 // item.CreatorIdAsUuid = new UUID(item.CreatorId);
461  }
462 
463  item.Owner = m_userInfo.PrincipalID;
464 
465  // Reset folder ID to the one in which we want to load it
466  item.Folder = loadFolder.ID;
467 
468  // Record the creator id for the item's asset so that we can use it later, if necessary, when the asset
469  // is loaded.
470  // FIXME: This relies on the items coming before the assets in the TAR file. Need to create stronger
471  // checks for this, and maybe even an external tool for creating OARs which enforces this, rather than
472  // relying on native tar tools.
473  m_creatorIdForAssetId[item.AssetID] = item.CreatorIdAsUuid;
474 
475  if (!m_InventoryService.AddItem(item))
476  m_log.WarnFormat("[INVENTORY ARCHIVER]: Unable to save item {0} in folder {1}", item.Name, item.Folder);
477 
478  return item;
479  }
480 
487  private bool LoadAsset(string assetPath, byte[] data)
488  {
489  //IRegionSerialiser serialiser = scene.RequestModuleInterface<IRegionSerialiser>();
490  // Right now we're nastily obtaining the UUID from the filename
491  string filename = assetPath.Remove(0, ArchiveConstants.ASSETS_PATH.Length);
492  int i = filename.LastIndexOf(ArchiveConstants.ASSET_EXTENSION_SEPARATOR);
493 
494  if (i == -1)
495  {
496  m_log.ErrorFormat(
497  "[INVENTORY ARCHIVER]: Could not find extension information in asset path {0} since it's missing the separator {1}. Skipping",
498  assetPath, ArchiveConstants.ASSET_EXTENSION_SEPARATOR);
499 
500  return false;
501  }
502 
503  string extension = filename.Substring(i);
504  string rawUuid = filename.Remove(filename.Length - extension.Length);
505  UUID assetId = new UUID(rawUuid);
506 
507  if (ArchiveConstants.EXTENSION_TO_ASSET_TYPE.ContainsKey(extension))
508  {
509  sbyte assetType = ArchiveConstants.EXTENSION_TO_ASSET_TYPE[extension];
510 
511  if (assetType == (sbyte)AssetType.Unknown)
512  {
513  m_log.WarnFormat("[INVENTORY ARCHIVER]: Importing {0} byte asset {1} with unknown type", data.Length, assetId);
514  }
515  else if (assetType == (sbyte)AssetType.Object)
516  {
517  if (m_creatorIdForAssetId.ContainsKey(assetId))
518  {
519  data = SceneObjectSerializer.ModifySerializedObject(assetId, data,
520  sog => {
521  bool modified = false;
522 
523  foreach (SceneObjectPart sop in sog.Parts)
524  {
525  if (string.IsNullOrEmpty(sop.CreatorData))
526  {
527  sop.CreatorID = m_creatorIdForAssetId[assetId];
528  modified = true;
529  }
530  }
531 
532  return modified;
533  });
534 
535  if (data == null)
536  return false;
537  }
538  }
539 
540  //m_log.DebugFormat("[INVENTORY ARCHIVER]: Importing asset {0}, type {1}", uuid, assetType);
541 
542  AssetBase asset = new AssetBase(assetId, "From IAR", assetType, UUID.Zero.ToString());
543  asset.Data = data;
544 
545  m_AssetService.Store(asset);
546 
547  return true;
548  }
549  else
550  {
551  m_log.ErrorFormat(
552  "[INVENTORY ARCHIVER]: Tried to dearchive data with path {0} with an unknown type extension {1}",
553  assetPath, extension);
554 
555  return false;
556  }
557  }
558 
564  public void LoadControlFile(string path, byte[] data)
565  {
566  XDocument doc = XDocument.Parse(Encoding.ASCII.GetString(data));
567  XElement archiveElement = doc.Element("archive");
568  int majorVersion = int.Parse(archiveElement.Attribute("major_version").Value);
569  int minorVersion = int.Parse(archiveElement.Attribute("minor_version").Value);
570  string version = string.Format("{0}.{1}", majorVersion, minorVersion);
571 
572  if (majorVersion > MAX_MAJOR_VERSION)
573  {
574  throw new Exception(
575  string.Format(
576  "The IAR you are trying to load has major version number of {0} but this version of OpenSim can only load IARs with major version number {1} and below",
577  majorVersion, MAX_MAJOR_VERSION));
578  }
579 
580  ControlFileLoaded = true;
581  m_log.InfoFormat("[INVENTORY ARCHIVER]: Loading IAR with version {0}", version);
582  }
583 
590  protected void LoadInventoryFile(string path, TarArchiveReader.TarEntryType entryType, byte[] data)
591  {
592  if (!ControlFileLoaded)
593  throw new Exception(
594  string.Format(
595  "The IAR you are trying to load does not list {0} before {1}. Aborting load",
597 
598  if (m_assetsLoaded)
599  throw new Exception(
600  string.Format(
601  "The IAR you are trying to load does not list all {0} before {1}. Aborting load",
603 
604  path = path.Substring(ArchiveConstants.INVENTORY_PATH.Length);
605 
606  // Trim off the file portion if we aren't already dealing with a directory path
607  if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY != entryType)
608  path = path.Remove(path.LastIndexOf("/") + 1);
609 
610  InventoryFolderBase foundFolder
611  = ReplicateArchivePathToUserInventory(
612  path, m_rootDestinationFolder, m_resolvedFolders, m_loadedNodes);
613 
614  if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY != entryType)
615  {
616  InventoryItemBase item = LoadItem(data, foundFolder);
617 
618  if (item != null)
619  {
620  m_successfulItemRestores++;
621 
622  // If we aren't loading the folder containing the item then well need to update the
623  // viewer separately for that item.
624  if (!m_loadedNodes.Contains(foundFolder))
625  m_loadedNodes.Add(item);
626  }
627  }
628 
629  m_inventoryNodesLoaded = true;
630  }
631 
637  protected void LoadAssetFile(string path, byte[] data)
638  {
639  if (!ControlFileLoaded)
640  throw new Exception(
641  string.Format(
642  "The IAR you are trying to load does not list {0} before {1}. Aborting load",
644 
645  if (!m_inventoryNodesLoaded)
646  throw new Exception(
647  string.Format(
648  "The IAR you are trying to load does not list all {0} before {1}. Aborting load",
650 
651  if (LoadAsset(path, data))
652  m_successfulAssetRestores++;
653  else
654  m_failedAssetRestores++;
655 
656  if ((m_successfulAssetRestores) % 50 == 0)
657  m_log.DebugFormat(
658  "[INVENTORY ARCHIVER]: Loaded {0} assets...",
659  m_successfulAssetRestores);
660 
661  m_assetsLoaded = true;
662  }
663  }
664 }
InventoryArchiveReadRequest(UUID id, InventoryArchiverModule module, IInventoryService inv, IAssetService assets, IUserAccountService uacc, UserAccount userInfo, string invPath, string loadPath, bool merge)
Temporary code to do the bare minimum required to read a tar archive for our purposes ...
InventoryArchiveReadRequest(IInventoryService inv, IAssetService assets, IUserAccountService uacc, UserAccount userInfo, string invPath, string loadPath, bool merge)
Constants for the archiving module
InventoryItemBase LoadItem(byte[] data, InventoryFolderBase loadFolder)
Load an item from the archive
InventoryFolderBase ReplicateArchivePathToUserInventory(string iarPath, InventoryFolderBase rootDestFolder, Dictionary< string, InventoryFolderBase > resolvedFolders, HashSet< InventoryNodeBase > loadedNodes)
Replicate the inventory paths in the archive to the user's inventory as necessary.
InventoryFolderBase ResolveDestinationFolder(InventoryFolderBase rootDestFolder, ref string archivePath, Dictionary< string, InventoryFolderBase > resolvedFolders)
Resolve a destination folder
OpenSim.Framework.Serialization.TarArchiveReader TarArchiveReader
InventoryArchiveReadRequest(UUID id, InventoryArchiverModule module, IInventoryService inv, IAssetService assets, IUserAccountService uacc, UserAccount userInfo, string invPath, Stream loadStream, bool merge)
static readonly IDictionary< string, sbyte > EXTENSION_TO_ASSET_TYPE
Ionic.Zlib.GZipStream GZipStream
Asset class. All Assets are reference by this class or a class derived from this class ...
Definition: AssetBase.cs:49
This module loads and saves OpenSimulator inventory archives
string CreatorData
Data about the creator in the form home_url;name
InventoryFolderBase m_rootDestinationFolder
Root destination folder for the IAR load.
void CreateFoldersForPath(InventoryFolderBase destFolder, string iarPathExisting, string iarPathToReplicate, Dictionary< string, InventoryFolderBase > resolvedFolders, HashSet< InventoryNodeBase > loadedNodes)
Create a set of folders for the given path.
Inventory Item - contains all the properties associated with an individual inventory piece...
string CreatorData
Extended creator information of the form <profile url>="">;<name>
void LoadInventoryFile(string path, TarArchiveReader.TarEntryType entryType, byte[] data)
Load inventory file
Interactive OpenSim region server
Definition: OpenSim.cs:55
virtual string Name
The name of the node (64 characters or less)
Ionic.Zlib.CompressionMode CompressionMode