[Scons-users] first builder / download

[Philipp Kraus]

On 2012-08-17 06:30:49 +0200, Philipp Kraus said:


> On 2012-08-15 09:58:36 +0200, Philipp Kraus said:

> 

>> On 2012-08-14 23:41:05 +0200, Gary Oberbrunner said:

>> 

>>> On Tue, Aug 14, 2012 at 5:32 PM, Philipp Kraus

>>> <philipp.kraus at flashpixx.de> wrote:

>>>> On 2012-08-14 00:16:04 +0200, Kraus Philipp said:

>>>> 

>>>>> Hi,

>>>>> 

>>>>> I try to create my first own builder for downloading a file. I read the

>>>>> documentation, but I don't find any good explain to do this.

>>>>> 

>>>>> This is my code

>>>>> 

>>>>> def url_downloadfile(target, source, env) :

>>>>> target = open( target, "wb" )

>>>>> f = urllib2.urlopen(source)

>>>>> target.write(f.read())

>>>>> target.close()

>>>>> f.close()

>>>>> return target, source

>>>>> 

>>>>> 

>>>>> def url_emitter(target, source, env) :

>>>>> if (target[0] == source[0]) :

>>>>> target[0] = str(source[0]).split("/")[-1]

>>>>> return target, source

>>>>> 

>>>>> 

>>>>> 

>>>>> dw = Builder(action = url_downloadfile, emitter = url_emitter,

>>>>> single_source = True )

>>>>> env = Environment(BUILDERS = {"Downloader" : dw})

>>>>> 

>>>>> env.Downloader( "http://myurl/myfile.tar.gz" )

>>>>> 

>>>>> 

>>>>> 

>>>>> I know that I can modify the source / target list with the emitter, so I

>>>>> think it is correct

>>>>> because the source list (here only on element) is the url, but the target

>>>>> list is here the url

>>>>> also, so I must modify the list, so that the last part of my url should be

>>>>> the target (or a target

>>>>> is setup manually, so I use this)

>>>>> 

>>>>> But now I get scons: *** [myfile.tar.gz] Source <url> not found, needed by

>>>>> target `myfile.tar.gz'.

>>>>> 

>>>>> This is imho correct, because the url is not a file, so Scons does not

>>>>> know anything about the url.

>>>>> How can I change this? I think, my input of the command is only a string

>>>>> object and not a FS Entry

>>>>> object

>>>> 

>>>> 

>>>> I have modified my source to this:

>>>> 

>>>> def url_downloadfile(target, source, env) :

>>>> target = open( str(target), "wb" )

>>>> f = urllib2.urlopen( str(source) )

>>>> target.write(f.read())

>>>> target.close()

>>>> f.close()

>>>> return None

>>>> 

>>>> 

>>>> def url_emitter(target, source, env) :

>>>> if not(isinstance(source[0], SCons.Node.Python.Value)) :

>>>> raise SCons.Errors.UserError("URL must be a value type, not '%s'" %

>>>> source[0].__class__)

>>>> 

>>>> if target[0] == source[0] :

>>>> target[0] = File( str(source[0]).split("/")[-1] )

>>>> 

>>>> return target, source

>>>> 

>>>> env = Environment( BUILDERS = { "Download" : Builder(action =

>>>> url_downloadfile, emitter = url_emitter, prefix = "http", single_source =

>>>> True, target_factory=File, source_factory=Value ) } )

>>>> 

>>>> If I run it, I get the message: scons: *** Do not know how to create a

>>>> target from source `http:/url/file.tar.gz'

>>>> Why I get this message? The emitter extracts the file of the url and change

>>>> the target

>>> 

>>> Using Value nodes is definitely the way to go.  You don't say how

>>> you're calling it: are you saying something like this:

>>>   tgt = Download(Value('http://url/file.tar.gz'))

>>> 

>>> ?

>> 

>> Yes,

>> env.Download( Value("http:/url/source.tar.gz") )

>> or

>> env.Download( "http:/url/source.tar.gz" )

>> or

>> env.Download( source = "http:/url/source.tar.gz" )

>> 

>> on each call the same error

> 

> I have modified the code to:

> 

> def DownloadFile(target, source, env) :

>     file   = open( str(source[0]), "wb" )

>     stream = urllib2.urlopen( str(target[0]) )

>     file.write(f.read())

>     file.close()

>     stream.close()

>     return None

> 

> 

> def url_emitter(target, source, env) :

>     if not(isinstance(source[0], SCons.Node.Python.Value)) :

>         raise SCons.Errors.UserError("URL must be a value type, not 

> '%s'" % source[0].__class__)

> 

>     #if target[0] == source[0] : *1*

>     #    target[0] = str(source[0]).split("/")[-1]

> 

>     # remove http prefix

>     target[0] = str(target[0]).replace("http", "")

> 

>     return target, source

> 

> I need remove the automatically file extraction, because this creates 

> the problem, that Scons creates the "Do not know how to create a target 

> from source" message.

> If I uncomment the *1* lines, scons run into the exception on the 

> Builder.py line 489 with this code:

>             try:

>                 t_from_s = slist[0].target_from_source

>             except AttributeError:

>                 raise UserError("Do not know how to create a target 

> from source `%s'" % slist[0])

> 

> 

> The next problem is, that Scons adds to the target name, if it is set, 

> the prefix "http", I don't understand it.

> 

> The builder call must be

> env = Environment( BUILDERS = { "Download" : Builder(action = 

> DownloadFile, emitter = url_emitter, prefix = "http", single_source = 

> True, target_factory=File, source_factory=Value ) } )

> env.Download( source="htt://url/ftp/lua-5.2.1.tar.gz", target="file.tar.gz" )

> 

> So in this case, I need the string replace in the emitter, because 

> source changes the file name to "httpfile.tar.gz"

> I need set the target and source parameters, but now the error "scons: 

> *** [file.tar.gz] http://url/file.tar.gz: No such file or directory" 

> occures.



This seems to be working:

------------
def url_print(s, target, source, env):
    print "downloading ["+str(source[0])+"] to ["+str(target[0])+"] ..."

def url_downloadfile(target, source, env) :
    file   = open( str(target[0]), "wb" )
    stream = urllib2.urlopen( str(source[0]) )
    file.write(stream.read())
    file.close()
    stream.close()
    return None

def url_emitter(target, source, env) :
    if not(isinstance(source[0], SCons.Node.Python.Value)) :
        raise SCons.Errors.UserError("URL must be a value type, not 
'%s'" % source[0].__class__)

    #if target[0] == source[0] :
    #    target[0] = str(source[0]).split("/")[-1]

    return target, source

def ParseAndDownload(env, parsefunction=None):
    if not(callable(parsefunction)) :
        raise SCons.Errors.UserError("parameter is a ['%s'] not a 
function " % parsefunction.__class__)

    [url, filename] = parsefunction()
    if (type(url) <> type("")) or (type(filename) <> type("")) or 
not(url) or not(filename):
        raise SCons.Errors.UserError("return parameter of the function 
must be string and need not be empty")

    return env.Download( filename, url )

AddMethod(Environment, ParseAndDownload)
env = Environment( BUILDERS = { "Download" : Builder(action = 
url_downloadfile, emitter = url_emitter, single_source = True, 
target_factory=File, source_factory=Value, 
PRINT_CMD_LINE_FUNC=url_print ) } )
------------------


For example I try to download the newest version of the LUA sources, so 
I create a function for parsing the HTML page source

def LUA_DownloadURL()  :
    # read download path of the LUA library (latest version)
    f = urllib2.urlopen("http://www.lua.org/download.html")
    html = f.read()
    f.close()

    found = re.search("<a href=\"ftp/lua-(.*)\.tar\.gz\">", html, 
re.IGNORECASE)
    if found == None :
        raise RuntimeError("LUA Download URL not found")

    downloadurl = found.group(0)

    downloadurl = downloadurl.replace("\"", "")
    downloadurl = downloadurl.replace("<", "")
    downloadurl = downloadurl.replace(">", "")
    downloadurl = re.sub(r'(?i)a href=', "", downloadurl)
    downloadurl = "http://www.lua.org/" + downloadurl

    filename    = downloadurl.split("/")[-1]

    return downloadurl, filename

The function must return first the URL of the download and on second 
the target filename, the main scons script shows:

x = env.ParseAndDownload( LUA_DownloadURL )

and in x is the target filename stored (actually lua-5.2.1.tar.gz)

Can you send me some tips to create better code?

Thanks

Phil