Implement FileSystem almost as MemoryFileSystemProvider.

Question

Hello, I'm trying to implement my own FileSystem. I bet you know :))

So I faced with few problems, do I understand correctly to implement something like Memory file system provider I need to create private variable with NodeBase type in CustomProvider class and store everything there? Or it's stored somewhere in base class? And, for example, for getting of children I need to get the data from base?

Sorry for retarding :)
Thanks.

Answer 1

Hi Dmitry,
I am afraid that one FileNode/DirectoryNode for file system is never enough. BTW: As you probably know, NodeBase is abstract class.
Quick and dirty simple memory provider. I hope that code speaks for itself.
Obvious warnings apply. Code has no production quality (e.g. provider is not thread safe) .

using System.Collections.Generic;
using System.IO;
using System.Linq;
using Rebex.IO.FileSystem;

namespace MemoryFs
{

    public class SimpleMemoryProvider
    : ReadWriteFileSystemProvider
{
    private readonly Dictionary<NodePath, NodeBase> _pathToNodeDictionary;
    private readonly Dictionary<NodeBase, FsNodeData> _storage;

    public SimpleMemoryProvider() : this(null)
    {
    }

    public SimpleMemoryProvider(FileSystemProviderSettings settings) : base(settings)
    {
        _pathToNodeDictionary = new Dictionary<NodePath, NodeBase>();
        _storage = new Dictionary<NodeBase, FsNodeData>();
    }

    public void AddRoot()
    {
        init();
    }

    private Dictionary<NodeBase, FsNodeData> Storage
    {
        get
        {
            return _storage;
        }
    }

    private Dictionary<NodePath, NodeBase> PathToNodeDictionary
    {
        get
        {
            return _pathToNodeDictionary;
        }
    }


    protected override NodeAttributes GetAttributes(NodeBase node)
    {
        if (!node.Exists())
        {
            return node.Attributes;
        }

        return Storage[node].Attributes;
    }

    protected override NodeTimeInfo GetTimeInfo(NodeBase node)
    {
        return Storage[node].TimeInfo;
    }

    protected override NodeBase GetChild(string name, DirectoryNode parent)
    {
        return Storage[parent].Children.FirstOrDefault(child => child.Name == name);
    }

    protected override IEnumerable<NodeBase> GetChildren(DirectoryNode parent, NodeType nodeType)
    {
        if (!parent.Exists())
        {
            return Enumerable.Empty<NodeBase>();
        }

        var children = Storage[parent].Children;
        return children;
    }

    protected override bool Exists(NodePath path, NodeType nodeType)
    {
        NodeBase node;
        PathToNodeDictionary.TryGetValue(path, out node);

        return node != null && node.NodeType == nodeType;
    }

    protected override long GetLength(NodeBase node)
    {
        if (!node.Exists())
        {
            return 0L;
        }

        return Storage[node].Length;
    }

    protected override NodeContent GetContent(NodeBase node, NodeContentParameters contentParameters)
    {
        if (!node.Exists())
        {
            return NodeContent.CreateEmptyContent();
        }

        return NodeContent.CreateDelayedWriteContent(Storage[node].Content.GetStream());
    }

    protected override DirectoryNode CreateDirectory(DirectoryNode parent, DirectoryNode child)
    {
        return createCommon(parent, child) as DirectoryNode;
    }

    protected override FileNode CreateFile(DirectoryNode parent, FileNode child)
    {
        return createCommon(parent, child) as FileNode;
    }

    protected override NodeBase Delete(NodeBase node)
    {
        if (!node.Exists())
        {
            return node;
        }

        Storage.Remove(node);
        PathToNodeDictionary.Remove(node.Path);
        Storage[node.Parent].Children.Remove(node);
        return node;
    }

    protected override NodeBase Rename(NodeBase node, string newName)
    {
        var isFile = node.NodeType == NodeType.File;
        var newNode = isFile
            ? (NodeBase) new FileNode(newName,
                node.Parent)
            : new DirectoryNode(newName, node.Parent);
        Delete(node);
        return newNode;
    }

    protected override NodeBase SaveContent(NodeBase node, NodeContent content)
    {
        if (!node.Exists())
        {
            return node;
        }

        Storage[node].Content = content;
        return node;
    }

    protected override NodeBase SetAttributes(NodeBase node, NodeAttributes attributes)
    {
        Storage[node].Attributes = attributes;
        return node;
    }

    protected override NodeBase SetTimeInfo(NodeBase node, NodeTimeInfo newTimeInfo)
    {
        Storage[node].TimeInfo = newTimeInfo;
        return node;
    }

    private void init()
    {
        if (_storage.Count == 0)
        {
            _storage.Add(Root, new FsNodeData());
            _pathToNodeDictionary.Add(Root.Path, Root);
        }
    }

    private NodeBase createCommon(DirectoryNode parent, NodeBase child)
    {
        Storage.Add(child, new FsNodeData());
        Storage[parent].Children.Add(child);
        PathToNodeDictionary.Add(child.Path, child);

        return child;
    }
}

internal class FsNodeData
{
    public FsNodeData()
    {
        Content = NodeContent.CreateEmptyContent();
        TimeInfo = new NodeTimeInfo();
        Children = new List<NodeBase>();
        Attributes = new NodeAttributes(FileAttributes.Offline);
    }

    public NodeAttributes Attributes
    {
        get;
        set;
    }

    public NodeTimeInfo TimeInfo
    {
        get;
        set;
    }

    public List<NodeBase> Children
    {
        get;
        set;
    }

    public long Length
    {
        get
        {
            return Content.GetStream().Length;
        }
    }

    public NodeContent Content { get; set; }
}

}

 //Usage:
    var provider  = new SimpleMemoryProvider();
    provider.AddRoot();

Comments

Thanks for sample. But saving of file isn't working and I don't understand one thing. :) How the file stream should be transfered to private storage and being used in SaveContent method?

In CreateCommon method it's only creating of elements in dictionary with an empty FsNodeData structure with empty Content property.
After the system is going to GetContent where it needs to get the stream from already created file or from storage, but in storage the Content property is empty.
Could you help me with that, please?

Maybe I missed some method in base class? or in NodeBase class?

---

Hi dmitry,  I am not sure, if I understand what is the problem?

If you need help, then please.

1) Post your the compilable code of the FS provider here.

2) Describe the problem, expected behavior and observerd behavior.

3) Much more better. Post a unit test(s) that demonstrates problem.

Empty content in a new instance of the FsNodeData contains new and empty MemoryStream. What else should contain?  We don't have data.

1) You create file with empty content.

2) File server calls GetContent method. You return DelayedWriteContent.

From doc.
*Method creates read/delayed write content from the given stream. Use this method when following is desired/acceptable behavior:
Underlying stream does not support immediate writing to the file.
*When NodeContent previously returned from this method is disposed and underlying stream has been changed, then the method method SaveContent() in the ReadWriteFileSystemProvider is called.*
(http://help.rebex.net/##RebexTotalPack.chm/Html/M_Rebex_IO_FileSystem_NodeContent_CreateDelayedWriteContent_1_bca7c9fe.htm)

3) When the file is uploaded, the SaveContent method is called. Because file node stream is disposed/closed.

Inherited methods GetContent/SetContent in the FileNode calls FS provider. In other words - FileNode neither store data nor create data nor load data from the air. FileNode relies on the implementation of the SaveContent/GetContent methods in the FileSystemProvider instance.


**SaveContent method in FS provider saves the stream.  SaveStream, as name suggests, saves stream. In the FS provider above the provider saves uploaded content in the associated instance of the helper class FsNodeData**

So description "file stream should be transfered to private storage and *being used* in SaveContent method?" does not make sense to me? Used? How? Save the stream, this is the responsibility of the methodd SaveContent.

---

The main problem is the stream in GetContent and in SaveContent is empty.
I sent the file though SFTP client and in these methods the stream is empty.

GetContent method almost the same as you provided:
protected override NodeContent GetContent(NodeBase node, NodeContentParameters contentParameters)
        {
            if (node.Exists())
            {
                Stream stream = this.storage[node].Content.GetStream();
                return NodeContent.CreateDelayedWriteContent(stream);
            }

            return NodeContent.CreateImmediateWriteContent(Stream.Null);
        }

SetContent:
protected override NodeBase SaveContent(NodeBase node, NodeContent content)
        {
            Stream stream = content.GetStream();

            this.stagingService.Upload("7799B138-3F23-47E6-9DE0-902448921637", "F0118_ASAP.csv", stream).GetAwaiter().GetResult();

            if (node.Exists())
            {
                //this.storage[node].Content = content;

                return this.Delete(node);
            }

            return node;
        }

the stream variable here is empty.

---

What is "empty"?
What is the value of the stream.Length property at the beginning and at the end of the method SaveContent?
Did you try reset position of the stream after upload (  this.stagingService.Upload("7799B138-3F23-47E6-9DE0-902448921637", "F0118_ASAP.csv", stream).GetAwaiter().GetResult());

stream.Position = 0;

---

Empty means the length is 0.

---

Stream stream = content.GetStream();

here.

Don't see on this method, the length is 0 before this.stagingService.Upload(").

---

Did you try my  other suggestion? Your uploader probably reads the stream to the end.
If the stream has Length 0  at the beginning of the method, then:

1) You uploaded empty file.
2) Upload was unsuccessful.

And please do not delete Node in SaveContent method.

---

1) Is impossible, it was working fine for MemoryFS.
2) Right, it wasn't successful, but why? Btw, you could see this behavior in implementation provided by you and see that uploading of file isn't working. I've tried just now. Just try to upload any file.

Ok, I won't delete :))

---

And please remove this code from GetContent method.
return NodeContent.CreateImmediateWriteContent(Stream.Null);

---

->And please remove this code from GetContent method.
->return NodeContent.CreateImmediateWriteContent(Stream.Null);

Ok, what to return then? It's from your sample. Btw, unfortunately, there's not such method in your lib "NodeContent.CreateEmptyContent();"

---

Also, if I use GetContentSurrogate event handler as you described a week early, such as:

//You can replace implementation of the GetContent method.
notifier.GetContentSurrogate += (sender, args) =>
{
    //analyze args.Node, args.ContentParameters and decide how to handle request
    if (args.Node.IsFile && openForWriteRequest(args))
    {
        var streamForWrite = new MemoryStream(); //or open your custom special server stream

        //Create the delayed write content. Delayed = I will save new content of the file later - when the upload of the file finished.
        args.ResultContent = NodeContent.CreateDelayedWriteContent(streamForWrite);
    }
};

The stream in SetContent method, or event handler also is empty (length is 0).

---

using System.Collections.Generic;
using System.IO;
using System.Linq;
using Rebex.IO.FileSystem;

namespace MemoryFs
{

    public class SimpleMemoryProvider
    : ReadWriteFileSystemProvider
{
    private readonly Dictionary<NodePath, NodeBase> _pathToNodeDictionary;
    private readonly Dictionary<NodeBase, FsNodeData> _storage;

    public SimpleMemoryProvider() : this(null)
    {
    }

    public SimpleMemoryProvider(FileSystemProviderSettings settings) : base(settings)
    {
        _pathToNodeDictionary = new Dictionary<NodePath, NodeBase>();
        _storage = new Dictionary<NodeBase, FsNodeData>();
    }

    public void AddRoot()
    {
        init();
    }

    private Dictionary<NodeBase, FsNodeData> Storage
    {
        get
        {
            return _storage;
        }
    }

    private Dictionary<NodePath, NodeBase> PathToNodeDictionary
    {
        get
        {
            return _pathToNodeDictionary;
        }
    }


    protected override NodeAttributes GetAttributes(NodeBase node)
    {
        if (!node.Exists())
        {
            return node.Attributes;
        }

        return Storage[node].Attributes;
    }

    protected override NodeTimeInfo GetTimeInfo(NodeBase node)
    {
        return Storage[node].TimeInfo;
    }

    protected override NodeBase GetChild(string name, DirectoryNode parent)
    {
        return Storage[parent].Children.FirstOrDefault(child => child.Name == name);
    }

    protected override IEnumerable<NodeBase> GetChildren(DirectoryNode parent, NodeType nodeType)
    {
        if (!parent.Exists())
        {
            return Enumerable.Empty<NodeBase>();
        }

        var children = Storage[parent].Children;
        return children;
    }

    protected override bool Exists(NodePath path, NodeType nodeType)
    {
        NodeBase node;
        PathToNodeDictionary.TryGetValue(path, out node);

        return node != null && node.NodeType == nodeType;
    }

    protected override long GetLength(NodeBase node)
    {
        if (!node.Exists())
        {
            return 0L;
        }

        return Storage[node].Length;
    }

    protected override NodeContent GetContent(NodeBase node, NodeContentParameters contentParameters)
    {
        if (!node.Exists())
        {
            return NodeContent.CreateDelayedWriteContent(new MemoryStream());
        }

        return NodeContent.CreateDelayedWriteContent(Storage[node].Content.GetStream());
    }

    protected override DirectoryNode CreateDirectory(DirectoryNode parent, DirectoryNode child)
    {
        return createCommon(parent, child) as DirectoryNode;
    }

    protected override FileNode CreateFile(DirectoryNode parent, FileNode child)
    {
        return createCommon(parent, child) as FileNode;
    }

    protected override NodeBase Delete(NodeBase node)
    {
        if (!node.Exists())
        {
            return node;
        }

        Storage.Remove(node);
        PathToNodeDictionary.Remove(node.Path);
        Storage[node.Parent].Children.Remove(node);
        return node;
    }

    protected override NodeBase Rename(NodeBase node, string newName)
    {
        var isFile = node.NodeType == NodeType.File;
        var newNode = isFile
            ? (NodeBase) new FileNode(newName,
                node.Parent)
            : new DirectoryNode(newName, node.Parent);
        Delete(node);
        return newNode;
    }

    protected override NodeBase SaveContent(NodeBase node, NodeContent content)
    {
        if (!node.Exists())
        {
            return node;
        }

        var myCurrentStream = new MemoryStream();
        content.GetStream().CopyTo(myCurrentStream);
        Storage[node].Content = NodeContent.CreateDelayedWriteContent(myCurrentStream);
        return node;
    }

    protected override NodeBase SetAttributes(NodeBase node, NodeAttributes attributes)
    {
        Storage[node].Attributes = attributes;
        return node;
    }

    protected override NodeBase SetTimeInfo(NodeBase node, NodeTimeInfo newTimeInfo)
    {
        Storage[node].TimeInfo = newTimeInfo;
        return node;
    }

    private void init()
    {
        if (_storage.Count == 0)
        {
            _storage.Add(Root, new FsNodeData());
            _pathToNodeDictionary.Add(Root.Path, Root);
        }
    }

    private NodeBase createCommon(DirectoryNode parent, NodeBase child)
    {
        Storage.Add(child, new FsNodeData());
        Storage[parent].Children.Add(child);
        PathToNodeDictionary.Add(child.Path, child);

        return child;
    }
}

internal class FsNodeData
{
    public FsNodeData()
    {
        Content = NodeContent.CreateDelayedWriteContent(new MemoryStream());
        TimeInfo = new NodeTimeInfo();
        Children = new List<NodeBase>();
        Attributes = new NodeAttributes(FileAttributes.Offline);
    }

    public NodeAttributes Attributes
    {
        get;
        set;
    }

    public NodeTimeInfo TimeInfo
    {
        get;
        set;
    }

    public List<NodeBase> Children
    {
        get;
        set;
    }

    public long Length
    {
        get
        {
            return Content.GetStream().Length;
        }
    }

    public NodeContent Content { get; set; }
}
}

---

Yes, you are right, CreateEmptyContent is our internal method, sorry for confusing.
Please try the version above, it works for me.

---

Omg, the reason was to not use Stream.Null. ..

Thanks for answer a lot!

---

dmitry, you are welcome, please use this more stable and safe version.

using System.Collections.Generic;
using System.IO;
using System.Linq;
using Rebex.IO.FileSystem;

namespace MemoryFs
{

    public class SimpleMemoryProvider
    : ReadWriteFileSystemProvider
    {
        private readonly Dictionary<NodePath, NodeBase> _pathToNodeDictionary;
        private readonly Dictionary<NodeBase, FsNodeData> _storage;

        public SimpleMemoryProvider() : this(null)
        {
        }

        public SimpleMemoryProvider(FileSystemProviderSettings settings) : base(settings)
        {
            _pathToNodeDictionary = new Dictionary<NodePath, NodeBase>();
            _storage = new Dictionary<NodeBase, FsNodeData>();
        }

        public void AddRoot()
        {
            init();
        }

        private Dictionary<NodeBase, FsNodeData> Storage
        {
            get
            {
                return _storage;
            }
        }

        private Dictionary<NodePath, NodeBase> PathToNodeDictionary
        {
            get
            {
                return _pathToNodeDictionary;
            }
        }


        protected override NodeAttributes GetAttributes(NodeBase node)
        {
            if (!node.Exists())
            {
                return node.Attributes;
            }

            return Storage[node].Attributes;
        }

        protected override NodeTimeInfo GetTimeInfo(NodeBase node)
        {
            return Storage[node].TimeInfo;
        }

        protected override NodeBase GetChild(string name, DirectoryNode parent)
        {
            return Storage[parent].Children.FirstOrDefault(child => child.Name == name);
        }

        protected override IEnumerable<NodeBase> GetChildren(DirectoryNode parent, NodeType nodeType)
        {
            if (!parent.Exists())
            {
                return Enumerable.Empty<NodeBase>();
            }

            var children = Storage[parent].Children;
            return children;
        }

        protected override bool Exists(NodePath path, NodeType nodeType)
        {
            NodeBase node;
            PathToNodeDictionary.TryGetValue(path, out node);

            return node != null && node.NodeType == nodeType;
        }

        protected override long GetLength(NodeBase node)
        {
            if (!node.Exists())
            {
                return 0L;
            }

            return Storage[node].Length;
        }

        protected override NodeContent GetContent(NodeBase node, NodeContentParameters contentParameters)
        {
            if (!node.Exists())
            {
                //error
                return NodeContent.CreateDelayedWriteContent(new MemoryStream());
            }

            var retStream = new MemoryStream();
            Storage[node].Content.CopyTo(retStream);
            retStream.Position = 0;
            Storage[node].Content.Position = 0;
            return contentParameters.AccessType == NodeContentAccess.Read
                ? NodeContent.CreateReadOnlyContent(retStream)
                : NodeContent.CreateDelayedWriteContent(retStream);
        }

        protected override DirectoryNode CreateDirectory(DirectoryNode parent, DirectoryNode child)
        {
            return createCommon(parent, child) as DirectoryNode;
        }

        protected override FileNode CreateFile(DirectoryNode parent, FileNode child)
        {
            return createCommon(parent, child) as FileNode;
        }

        protected override NodeBase Delete(NodeBase node)
        {
            if (!node.Exists())
            {
                return node;
            }

            Storage.Remove(node);
            PathToNodeDictionary.Remove(node.Path);
            Storage[node.Parent].Children.Remove(node);
            return node;
        }

        protected override NodeBase Rename(NodeBase node, string newName)
        {
            var isFile = node.NodeType == NodeType.File;
            var newNode = isFile
                ? (NodeBase)new FileNode(newName,
                    node.Parent)
                : new DirectoryNode(newName, node.Parent);
            Delete(node);
            return newNode;
        }

        protected override NodeBase SaveContent(NodeBase node, NodeContent content)
        {
            if (!node.Exists())
            {
                return node;
            }

            var myCurrentStream = new MemoryStream();
            content.GetStream().CopyTo(myCurrentStream);
            myCurrentStream.Position = 0;
            Storage[node].Content = myCurrentStream;
            return node;
        }

        protected override NodeBase SetAttributes(NodeBase node, NodeAttributes attributes)
        {
            Storage[node].Attributes = attributes;
            return node;
        }

        protected override NodeBase SetTimeInfo(NodeBase node, NodeTimeInfo newTimeInfo)
        {
            Storage[node].TimeInfo = newTimeInfo;
            return node;
        }

        private void init()
        {
            if (_storage.Count == 0)
            {
                _storage.Add(Root, new FsNodeData());
                _pathToNodeDictionary.Add(Root.Path, Root);
            }
        }

        private NodeBase createCommon(DirectoryNode parent, NodeBase child)
        {
            Storage.Add(child, new FsNodeData());
            Storage[parent].Children.Add(child);
            PathToNodeDictionary.Add(child.Path, child);

            return child;
        }
    }

    internal class FsNodeData
    {
        public FsNodeData()
        {
            Content = new MemoryStream();
            TimeInfo = new NodeTimeInfo();
            Children = new List<NodeBase>();
            Attributes = new NodeAttributes(FileAttributes.Offline);
        }

        public NodeAttributes Attributes
        {
            get;
            set;
        }

        public NodeTimeInfo TimeInfo
        {
            get;
            set;
        }

        public List<NodeBase> Children
        {
            get;
            set;
        }

        public long Length
        {
            get
            {
                return Content.Length;
            }
        }

        public MemoryStream Content { get; set; }
    }
}

//Usage: