[Scons-users] first builder / download

Philipp Kraus philipp.kraus at flashpixx.de
Fri Aug 17 01:42:06 EDT 2012


On 2012-08-17 06:30:49 +0200, Philipp Kraus said:


> On 2012-08-15 09:58:36 +0200, Philipp Kraus said:

>

>> On 2012-08-14 23:41:05 +0200, Gary Oberbrunner said:

>>

>>> On Tue, Aug 14, 2012 at 5:32 PM, Philipp Kraus

>>> <philipp.kraus at flashpixx.de> wrote:

>>>> On 2012-08-14 00:16:04 +0200, Kraus Philipp said:

>>>>

>>>>> Hi,

>>>>>

>>>>> I try to create my first own builder for downloading a file. I read the

>>>>> documentation, but I don't find any good explain to do this.

>>>>>

>>>>> This is my code

>>>>>

>>>>> def url_downloadfile(target, source, env) :

>>>>> target = open( target, "wb" )

>>>>> f = urllib2.urlopen(source)

>>>>> target.write(f.read())

>>>>> target.close()

>>>>> f.close()

>>>>> return target, source

>>>>>

>>>>>

>>>>> def url_emitter(target, source, env) :

>>>>> if (target[0] == source[0]) :

>>>>> target[0] = str(source[0]).split("/")[-1]

>>>>> return target, source

>>>>>

>>>>>

>>>>>

>>>>> dw = Builder(action = url_downloadfile, emitter = url_emitter,

>>>>> single_source = True )

>>>>> env = Environment(BUILDERS = {"Downloader" : dw})

>>>>>

>>>>> env.Downloader( "http://myurl/myfile.tar.gz" )

>>>>>

>>>>>

>>>>>

>>>>> I know that I can modify the source / target list with the emitter, so I

>>>>> think it is correct

>>>>> because the source list (here only on element) is the url, but the target

>>>>> list is here the url

>>>>> also, so I must modify the list, so that the last part of my url should be

>>>>> the target (or a target

>>>>> is setup manually, so I use this)

>>>>>

>>>>> But now I get scons: *** [myfile.tar.gz] Source <url> not found, needed by

>>>>> target `myfile.tar.gz'.

>>>>>

>>>>> This is imho correct, because the url is not a file, so Scons does not

>>>>> know anything about the url.

>>>>> How can I change this? I think, my input of the command is only a string

>>>>> object and not a FS Entry

>>>>> object

>>>>

>>>>

>>>> I have modified my source to this:

>>>>

>>>> def url_downloadfile(target, source, env) :

>>>> target = open( str(target), "wb" )

>>>> f = urllib2.urlopen( str(source) )

>>>> target.write(f.read())

>>>> target.close()

>>>> f.close()

>>>> return None

>>>>

>>>>

>>>> def url_emitter(target, source, env) :

>>>> if not(isinstance(source[0], SCons.Node.Python.Value)) :

>>>> raise SCons.Errors.UserError("URL must be a value type, not '%s'" %

>>>> source[0].__class__)

>>>>

>>>> if target[0] == source[0] :

>>>> target[0] = File( str(source[0]).split("/")[-1] )

>>>>

>>>> return target, source

>>>>

>>>> env = Environment( BUILDERS = { "Download" : Builder(action =

>>>> url_downloadfile, emitter = url_emitter, prefix = "http", single_source =

>>>> True, target_factory=File, source_factory=Value ) } )

>>>>

>>>> If I run it, I get the message: scons: *** Do not know how to create a

>>>> target from source `http:/url/file.tar.gz'

>>>> Why I get this message? The emitter extracts the file of the url and change

>>>> the target

>>>

>>> Using Value nodes is definitely the way to go. You don't say how

>>> you're calling it: are you saying something like this:

>>> tgt = Download(Value('http://url/file.tar.gz'))

>>>

>>> ?

>>

>> Yes,

>> env.Download( Value("http:/url/source.tar.gz") )

>> or

>> env.Download( "http:/url/source.tar.gz" )

>> or

>> env.Download( source = "http:/url/source.tar.gz" )

>>

>> on each call the same error

>

> I have modified the code to:

>

> def DownloadFile(target, source, env) :

> file = open( str(source[0]), "wb" )

> stream = urllib2.urlopen( str(target[0]) )

> file.write(f.read())

> file.close()

> stream.close()

> return None

>

>

> def url_emitter(target, source, env) :

> if not(isinstance(source[0], SCons.Node.Python.Value)) :

> raise SCons.Errors.UserError("URL must be a value type, not

> '%s'" % source[0].__class__)

>

> #if target[0] == source[0] : *1*

> # target[0] = str(source[0]).split("/")[-1]

>

> # remove http prefix

> target[0] = str(target[0]).replace("http", "")

>

> return target, source

>

> I need remove the automatically file extraction, because this creates

> the problem, that Scons creates the "Do not know how to create a target

> from source" message.

> If I uncomment the *1* lines, scons run into the exception on the

> Builder.py line 489 with this code:

> try:

> t_from_s = slist[0].target_from_source

> except AttributeError:

> raise UserError("Do not know how to create a target

> from source `%s'" % slist[0])

>

>

> The next problem is, that Scons adds to the target name, if it is set,

> the prefix "http", I don't understand it.

>

> The builder call must be

> env = Environment( BUILDERS = { "Download" : Builder(action =

> DownloadFile, emitter = url_emitter, prefix = "http", single_source =

> True, target_factory=File, source_factory=Value ) } )

> env.Download( source="htt://url/ftp/lua-5.2.1.tar.gz", target="file.tar.gz" )

>

> So in this case, I need the string replace in the emitter, because

> source changes the file name to "httpfile.tar.gz"

> I need set the target and source parameters, but now the error "scons:

> *** [file.tar.gz] http://url/file.tar.gz: No such file or directory"

> occures.



This seems to be working:

------------
def url_print(s, target, source, env):
print "downloading ["+str(source[0])+"] to ["+str(target[0])+"] ..."

def url_downloadfile(target, source, env) :
file = open( str(target[0]), "wb" )
stream = urllib2.urlopen( str(source[0]) )
file.write(stream.read())
file.close()
stream.close()
return None

def url_emitter(target, source, env) :
if not(isinstance(source[0], SCons.Node.Python.Value)) :
raise SCons.Errors.UserError("URL must be a value type, not
'%s'" % source[0].__class__)

#if target[0] == source[0] :
# target[0] = str(source[0]).split("/")[-1]

return target, source

def ParseAndDownload(env, parsefunction=None):
if not(callable(parsefunction)) :
raise SCons.Errors.UserError("parameter is a ['%s'] not a
function " % parsefunction.__class__)

[url, filename] = parsefunction()
if (type(url) <> type("")) or (type(filename) <> type("")) or
not(url) or not(filename):
raise SCons.Errors.UserError("return parameter of the function
must be string and need not be empty")

return env.Download( filename, url )

AddMethod(Environment, ParseAndDownload)
env = Environment( BUILDERS = { "Download" : Builder(action =
url_downloadfile, emitter = url_emitter, single_source = True,
target_factory=File, source_factory=Value,
PRINT_CMD_LINE_FUNC=url_print ) } )
------------------


For example I try to download the newest version of the LUA sources, so
I create a function for parsing the HTML page source

def LUA_DownloadURL() :
# read download path of the LUA library (latest version)
f = urllib2.urlopen("http://www.lua.org/download.html")
html = f.read()
f.close()

found = re.search("<a href=\"ftp/lua-(.*)\.tar\.gz\">", html,
re.IGNORECASE)
if found == None :
raise RuntimeError("LUA Download URL not found")

downloadurl = found.group(0)

downloadurl = downloadurl.replace("\"", "")
downloadurl = downloadurl.replace("<", "")
downloadurl = downloadurl.replace(">", "")
downloadurl = re.sub(r'(?i)a href=', "", downloadurl)
downloadurl = "http://www.lua.org/" + downloadurl

filename = downloadurl.split("/")[-1]

return downloadurl, filename

The function must return first the URL of the download and on second
the target filename, the main scons script shows:

x = env.ParseAndDownload( LUA_DownloadURL )

and in x is the target filename stored (actually lua-5.2.1.tar.gz)

Can you send me some tips to create better code?

Thanks

Phil




More information about the Scons-users mailing list