diff --git a/lib/net/scp/download.rb b/lib/net/scp/download.rb
index 22aced3..8754c39 100644
--- a/lib/net/scp/download.rb
+++ b/lib/net/scp/download.rb
@@ -50,8 +50,11 @@ def read_directive_state(channel)
       when :file
         read_file(channel, directive)
       when :end
+        directory = channel[:stack].pop
+        if channel[:options][:preserve] && directory && directory[:times]
+          File.utime(directory[:times][:atime], directory[:times][:mtime], channel[:local])
+        end
         channel[:local] = File.dirname(channel[:local])
-        channel[:stack].pop
         channel[:state] = :finish if channel[:stack].empty?
       end
 
@@ -134,11 +137,7 @@ def read_directory(channel, directive)
         Dir.mkdir(channel[:local], directive[:mode] | 0700)
       end
 
-      if channel[:options][:preserve] && channel[:times]
-        File.utime(channel[:times][:atime], channel[:times][:mtime], channel[:local])
-      end
-
-      channel[:stack] << directive
+      channel[:stack] << directive.merge(:times => channel[:times])
       channel[:times] = nil
     end