Python: Current directory in an os.walk

I need to get the current directory in an os.walk process. It works when there is just one subdirectory level but fails when there's more. Please advise...

[CODE]

# AFFECTS everything reachable from the directory named in "top",
# assuming there are no symbolic links.
# CAUTION:  This is dangerous!  For example, if top == '/', it
# could affect all your disk files.

import os, glob, arcpy, csv, sys, shutil, datetime
top = r'L:\Raster_Data\Topographic_Maps'
RootOutput = r'L:\Raster_Data\Topographic_Maps'
#FileList = csv.reader(open('FileList.csv'))
SearchString=['Temp_Pol', 'Spatial_Ex']

filecount=0
successcount=0
errorcount=0

print "Working in: "+os.getcwd()

list =[]
f = open(RootOutput+'\\Success_LOG.txt', 'a')
f.write("Log of files Succesfully processed. RESULT of process run @:"+str(datetime.datetime.now())+"\n")
f.close()

#for File in FileList:
for root, dirs, files in os.walk(top, topdown=False):
  #for directory in dirs:
    for file in files:
      #currentPath=os.path.join(root,directory)
      currentPath=os.path.abspath(file)
      os.chdir(currentPath)
      #arcpy.env.workspace = currentPath
      #print os.getcwd()
      lstFCs = glob.glob('*'+SearchString[0]+'*.shp')
      #print lstFCs
      OutPutDir=os.path.abspath(currentPath)
      for fc in lstFCs:
          filecount=filecount+1
          list.append(OutPutDir+"\\"+fc)       

      lstFCs = glob.glob('*'+SearchString[1]+'*.shp')
      #print lstFCs
      for fc in lstFCs:
          OutPutDir=RootOutput+"\\"+directory
          filecount=filecount+1
          list.append(OutPutDir+"\\"+fc)

print 'Merging: ' + str(list)
#arcpy.Merge_management(list, RootOutput+"\\Full_Extent.shp")
print 'Created: '+RootOutput+"\\Full_Extent.shp"
f = open(RootOutput+'\\Success_LOG.txt', 'a')
f.write(str(list)+"\n\n Merged to: "+RootOutput+"\\Full_Extent.shp")
f.close()

so the list should be appended with the fc and the full path to it but just gets the root path and the final part of the path -not the directories in between.

Thanks for your advise,

[Error Messages]

Working in: L:\Raster_Data\Topographic_Maps Merging: ['L:\Raster_Data\Topographic_Maps\100K\2010_100K\Map_Sheets_BestResolution\qld_north\SE55\ecw\prj_Temp_Polygon_Extent_0.shp', 'L:\Raster_Data\Topographic_Maps\100K\2010_100K\Map_Sheets_BestResolution\qld_north\SE55\ecw\Temp_Polygon_Extent_0.shp', 'L:\Raster_Data\Topographic_Maps\ecw\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SC54\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SC55\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SD54\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SD55\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SE54\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\100K\2010_100K\Map_Sheets_BestResolution\qld_north\SE55\prj_Temp_Polygon_Extent_0.shp', 'L:\Raster_Data\Topographic_Maps\100K\201开发者_C百科0_100K\Map_Sheets_BestResolution\qld_north\SE55\Temp_Polygon_Extent_0.shp', 'L:\Raster_Data\Topographic_Maps\SE55\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SF54\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SF55\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SF56\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SG55\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SG56\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\SH56\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\Tablelands_100K\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\200DPI\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\1M\prj_Temp_Polygon_Extent_0.shp', 'L:\Raster_Data\Topographic_Maps\1M\Temp_Polygon_Extent_0.shp', 'L:\Raster_Data\Topographic_Maps\250K\prj_Temp_Polygon_Extent_1.shp', 'L:\Raster_Data\Topographic_Maps\250K\Temp_Polygon_Extent_1.shp', 'L:\Raster_Data\Topographic_Maps\250K\Spatial_Extent.shp', 'L:\Raster_Data\Topographic_Maps\5M\prj_Temp_Polygon_Extent_2.shp', 'L:\Raster_Data\Topographic_Maps\5M\Temp_Polygon_Extent_2.shp', 'L:\Raster_Data\Topographic_Maps\5M\Spatial_Extent.shp'] Traceback (most recent call last): File "L:\Raster_Data\Topographic_Maps\CreateFileList.py", line 64, in arcpy.Merge_management(list, RootOutput+"\Full_Extent.shp")

File "C:\Program Files\ArcGIS\Desktop10.0\arcpy\arcpy\management.py", line 3124, in Merge raise e ExecuteError: Failed to execute. Parameters are not valid. ERROR 000732: Input Datasets: Dataset L:\Raster_Data\Topographic_Maps\100K\2010_100K\Map_Sheets_BestResolution\qld_north\SE55\ecw\prj_Temp_Polygon_Extent_0.shp;L:\Raster_Data\Topographic_Maps\100K\2010_100K\Map_Sheets_BestResolution\qld_north\SE55\ecw\Temp_Polygon_Extent_0.shp;L:\Raster_Data\Topographic_Maps\ecw\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SC54\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SC55\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SD54\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SD55\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SE54\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\100K\2010_100K\Map_Sheets_BestResolution\qld_north\SE55\prj_Temp_Polygon_Extent_0.shp;L:\Raster_Data\Topographic_Maps\100K\2010_100K\Map_Sheets_BestResolution\qld_north\SE55\Temp_Polygon_Extent_0.shp;L:\Raster_Data\Topographic_Maps\SE55\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SF54\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SF55\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SF56\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SG55\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SG56\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\SH56\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\Tablelands_100K\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\200DPI\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\1M\prj_Temp_Polygon_Extent_0.shp;L:\Raster_Data\Topographic_Maps\1M\Temp_Polygon_Extent_0.shp;L:\Raster_Data\Topographic_Maps\250K\prj_Temp_Polygon_Extent_1.shp;L:\Raster_Data\Topographic_Maps\250K\Temp_Polygon_Extent_1.shp;L:\Raster_Data\Topographic_Maps\250K\Spatial_Extent.shp;L:\Raster_Data\Topographic_Maps\5M\prj_Temp_Polygon_Extent_2.shp;L:\Raster_Data\Topographic_Maps\5M\Temp_Polygon_Extent_2.shp;L:\Raster_Data\Topographic_Maps\5M\Spatial_Extent.shp does not exist or is not supported Failed to execute (Merge).

Working in: L:\Raster_Data\Topographic_Maps Traceback (most recent call last): File "L:\Raster_Data\Topographic_Maps\CreateFileList.py", line 28, in os.chdir(currentPath) WindowsError: [Error 2] The system cannot find the file specified: 'L:\Raster_Data\Topographic_Maps\ecw\SC54' directory 'SC54' dirs ['SC54', 'SC55', 'SD54', 'SD55', 'SE54', 'SE55'] os.path.abspath(dirs[0]) 'L:\Raster_Data\Topographic_Maps\ecw\SC54' os.getcwd() 'L:\Raster_Data\Topographic_Maps\ecw' Working in: L:\Raster_Data\Topographic_Maps Traceback (most recent call last): File "L:\Raster_Data\Topographic_Maps\CreateFileList.py", line 28, in os.chdir(currentPath) WindowsError: [Error 2] The system cannot find the file specified: 'L:\Raster_Data\Topographic_Maps\7178cp_dd.ers' file '7178cp_dd.ers' os.path.abspath os.path.abspath(file) 'L:\Raster_Data\Topographic_Maps\7178cp_dd.ers'

---

Thanks all, I used the input from the forum to complete the script. It's below for anyone who wants it. best,

# AFFECTS everything reachable from the directory named in "top",
# assuming there are no symbolic links.
# CAUTION:  This is dangerous!  For example, if top == '/', it
# could affect all your disk files.

import os, arcpy, sys, datetime
top = os.getcwd()
RootOutput = top
FileTypes=['shp']
SearchStrings=['Temp_Pol', 'Spatial_Ex']

filecount=0
#successcount=0
#errorcount=0

print "Working in: "+os.getcwd()

list =[]
f = open(RootOutput+'\\Success_LOG.txt', 'a')
f.write("Log of files Succesfully processed. RESULT of process run @:"+str(datetime.datetime.now())+"\n")
f.close()

for root, dirs, files in os.walk(top, topdown=False):
    for fl in files:
      currentFile=os.path.join(root, fl)
      for FileType in FileTypes:
          status= str.endswith(currentFile,FileType)
          if str(status) == 'True':
              for SearchString in SearchStrings:
                  if str(SearchString in currentFile) == 'True':
                    #print str(currentFile)+str(status)       
                    filecount=filecount+1
                    list.append(currentFile)

print 'Merging: ' + str(list)

#Replace with any function you want to carry out on the generated list of files. #arcpy.Merge_management(list, RootOutput+"\Full_Extent.shp")

print 'Created: '+RootOutput+"\\Full_Extent.shp"
f = open(RootOutput+'\\Success_LOG.txt', 'a')
f.write(str(list)+"\n\n Merged to: "+RootOutput+"\\Full_Extent.shp")
f.close()

---

You should use

os.path.join(root, file) 

instead of simply using file like suggested in the os.walk doc examples os.walk

Btw, be careful with the reserved keywords. file is a built-in function and list too

>>> a = list()
>>> a
[]
>>> list = []
>>> b = list()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: 'list' object is not callable

---

For some applications we always need to change the current working directory while we are in os.walk recursive call, in that case I would suggest changing the current working directory twice as shown below. I am writing about situations where having absolute file paths will not help .

from os import listdir
from os.path import isfile, join
import os
import re
# store the location of the top most directory 
top = os.getcwd()

for (dirname, dirs, files) in os.walk(os.getcwd()):        
        for filename in files:
                os.chdir(dirname)
                # add all your operations for the current job in the directory
                # Now go back to the top of the chain
                os.chdir(top)

---

It looks like you're after a recursive glob. Something like the code below might be of use:

class rglob:
    '''A recursive/regex enhanced glob
       adapted from os-path-walk-example-3.py - http://effbot.org/librarybook/os-path.htm 
    '''
    def __init__(self, directory, pattern="*", regex=False, regex_flags=0, recurse=True):
        ''' @type    directory: C{str}
            @param   directory: Path to search
            @type    pattern: C{type}
            @param   pattern: Regular expression/wildcard pattern to match files against
            @type    regex: C{boolean}
            @param   regex: Use regular expression matching (if False, use fnmatch)
                            See U{http://docs.python.org/library/re.html}
            @type    regex_flags: C{int}
            @param   regex_flags: Flags to pass to the regular expression compiler.
                                  See U{http://docs.python.org/library/re.html}
            @type    recurse: C{boolean} 
            @param   recurse: Recurse into the directory?
        '''
        self.stack = [directory]
        self.pattern = pattern
        self.regex = regex
        self.recurse = recurse
        self.regex_flags = regex_flags
        self.files = []
        self.index = 0

    def __getitem__(self, index):
        while 1:
            try:
                file = self.files[self.index]
                self.index = self.index + 1
            except IndexError:
                # pop next directory from stack

                self.directory = self.stack.pop()
                try:
                    self.files = os.listdir(self.directory)
                    self.index = 0
                except:pass
            else:
                # got a filename
                fullname = os.path.join(self.directory, file)
                if os.path.isdir(fullname) and not os.path.islink(fullname) and self.recurse:
                    self.stack.append(fullname)
                if self.regex:
                    import re
                    if re.search(self.pattern,file,self.regex_flags):
                        return fullname
                else:
                    import fnmatch
                    if fnmatch.fnmatch(file, self.pattern):
                        return fullname

shplist=[shp for shp in rglob(top,'*.shp')]
print 'Merging: ' + str(shplist)
#arcpy.Merge_management(shplist, RootOutput+"\\Full_Extent.shp")
print 'Created: '+RootOutput+"\\Full_Extent.shp"
f = open(RootOutput+'\\Success_LOG.txt', 'a')
f.write(str(shplist)+"\n\n Merged to: "+RootOutput+"\\Full_Extent.shp")
f.close()