Difference: CMSLoadTestLHCONEScripts (1 vs. 9)

Revision 92011/06/02 - Main.JamesLetts

Line: 1 to 1
 
META TOPICPARENT name="CMSLoadTestLHCONEResults"

Scripts

Line: 24 to 24
  fail={} rate={} for node in dom.getElementsByTagName('transfer'):
Deleted:
<
<
done_files=node.attributes["done_files"].value fail_files=node.attributes["fail_files"].value
  time_bin=node.attributes["timebin"].value done[time_bin]=node.attributes["done_files"].value fail[time_bin]=node.attributes["fail_files"].value
Line: 46 to 44
  sumdone+=int(str(done[key])) sumfail+=int(str(fail[key]))
Deleted:
<
<
if ( sumdone >= int(str(nfiles)) ) :
  lastkey=int(str(key))
Added:
>
>
if ( sumdone >= int(str(nfiles)) ) :
  break if ( int(str(key)) > now ) : break

maxrate/=(1024.*1024) latency=float(int(lastkey)-int(starttime))/3600.

Deleted:
<
<
if latency < 0 : latency=-1
  try : quality=float(sumdone)/float(sumdone+sumfail)*100 except ZeroDivisionError? :

Revision 82011/05/31 - Main.JamesLetts

Line: 1 to 1
 
META TOPICPARENT name="CMSLoadTestLHCONEResults"

Scripts

Line: 13 to 13
 import xml.dom.minidom from xml.dom.minidom import Node
Changed:
<
<
def GetTransferResults? (fromsite,tosite,nfiles,starttime,delta=86400):
>
>
def GetTransferResults? (fromsite,tosite,nfiles,starttime):
  now=int(time.time())
Changed:
<
<
endtime=int(starttime)+int(delta) if ( endtime > now ) : endtime=now url="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=300&starttime="+str(starttime)+"&endtime="+str(endtime) print url
>
>
url="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=300&starttime="+str(starttime)+"&endtime="+str(now)
  xmldata = urllib2.urlopen(url).read() dom = xml.dom.minidom.parseString(xmldata)

done={} fail={}

Added:
>
>
rate={}
  for node in dom.getElementsByTagName('transfer'): done_files=node.attributes["done_files"].value fail_files=node.attributes["fail_files"].value time_bin=node.attributes["timebin"].value
Changed:
<
<
done[time_bin]=done_files fail[time_bin]=fail_files
>
>
done[time_bin]=node.attributes["done_files"].value fail[time_bin]=node.attributes["fail_files"].value rate[time_bin]=node.attributes["rate"].value
  sumdone=0 sumfail=0 lastkey=0
Added:
>
>
maxrate=0. moving_average=[0,0,0,0,0,0,0,0,0,0,0,0]
  for key in sorted(done.iterkeys()):
Added:
>
>
rate_5m=float(str(rate[key])) moving_average.pop(0) moving_average.append(rate_5m) avg_1h=sum(moving_average)/12. if ( avg_1h > maxrate ) : maxrate=avg_1h
  sumdone+=int(str(done[key])) sumfail+=int(str(fail[key])) if ( sumdone >= int(str(nfiles)) ) :
Line: 44 to 52
  if ( int(str(key)) > now ) : break
Deleted:
<
<
url2="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=3600&starttime="+str(starttime)+"&endtime="+str(lastkey) xmldata2 = urllib2.urlopen(url2).read() dom2 = xml.dom.minidom.parseString(xmldata2) maxrate=0. for node in dom2.getElementsByTagName('transfer'): rate=float(str(node.attributes["rate"].value)) if ( rate > maxrate ) : maxrate=rate
  maxrate/=(1024.*1024) latency=float(int(lastkey)-int(starttime))/3600. if latency < 0 :
Line: 77 to 76
 GetTransferResults? (fromsite,tosite,nfiles,starttime)
Deleted:
<
<
-- JamesLetts - 2011/05/21
 \ No newline at end of file
Added:
>
>
-- JamesLetts - 2011/05/31
 \ No newline at end of file

Revision 72011/05/26 - Main.JamesLetts

Line: 1 to 1
 
META TOPICPARENT name="CMSLoadTestLHCONEResults"

Scripts

Line: 13 to 13
 import xml.dom.minidom from xml.dom.minidom import Node
Deleted:
<
<
 def GetTransferResults? (fromsite,tosite,nfiles,starttime,delta=86400):
Changed:
<
<
url="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=300&starttime="+str(starttime)+"&endtime="+str(int(starttime)+int(delta))
>
>
now=int(time.time()) endtime=int(starttime)+int(delta) if ( endtime > now ) : endtime=now url="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=300&starttime="+str(starttime)+"&endtime="+str(endtime) print url
  xmldata = urllib2.urlopen(url).read() dom = xml.dom.minidom.parseString(xmldata)
Line: 31 to 35
  sumdone=0 sumfail=0 lastkey=0
Deleted:
<
<
now=int(time.time())
  for key in sorted(done.iterkeys()): sumdone+=int(str(done[key])) sumfail+=int(str(fail[key]))

Revision 62011/05/21 - Main.JamesLetts

Line: 1 to 1
 
META TOPICPARENT name="CMSLoadTestLHCONEResults"

Scripts

Line: 31 to 31
  sumdone=0 sumfail=0 lastkey=0
Changed:
<
<
now=time.ctime()
>
>
now=int(time.time())
  for key in sorted(done.iterkeys()): sumdone+=int(str(done[key])) sumfail+=int(str(fail[key]))
Changed:
<
<
if ( sumdone >= nfiles ) :
>
>
if ( sumdone >= int(str(nfiles)) ) :
  lastkey=int(str(key)) break
Changed:
<
<
if ( key > now ) :
>
>
if ( int(str(key)) > now ) :
  break

url2="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=3600&starttime="+str(starttime)+"&endtime="+str(lastkey)

Line: 59 to 59
  except ZeroDivisionError? : quality=-1
Changed:
<
<
print sumdone, "files completed" print("%.1f" % maxrate),"MiB/s" print("%.1f" % latency),"h" print("%.1f" % quality),"%"
>
>
print "Testing",fromsite,"to",tosite print "Number of files transfered......",sumdone print "Maximum hourly transfer rate....",("%.1f" % maxrate),"MiB/s" print "Latency.........................",("%.1f" % latency),"h" print "Quality transfer success rate...",("%.1f" % quality),"%"
  return
Line: 73 to 74
 GetTransferResults? (fromsite,tosite,nfiles,starttime)
Deleted:
<
<
-- JamesLetts - 2011/05/17
 \ No newline at end of file
Added:
>
>
-- JamesLetts - 2011/05/21
 \ No newline at end of file

Revision 52011/05/21 - Main.JamesLetts

Line: 1 to 1
 
META TOPICPARENT name="CMSLoadTestLHCONEResults"

Scripts

Line: 9 to 9
 # import sys,os.path import string,time
Changed:
<
<
import urllib
>
>
import urllib,urllib2
 import xml.dom.minidom from xml.dom.minidom import Node
Changed:
<
<
def GetTransferResults? (starttime,fromsite="T2_US_Wisconsin",tosite="T2_US_MIT",delta=86400): import urllib2, xml.dom.minidom
>
>
def GetTransferResults? (fromsite,tosite,nfiles,starttime,delta=86400):
  url="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=300&starttime="+str(starttime)+"&endtime="+str(int(starttime)+int(delta)) xmldata = urllib2.urlopen(url).read() dom = xml.dom.minidom.parseString(xmldata)
Line: 36 to 35
  for key in sorted(done.iterkeys()): sumdone+=int(str(done[key])) sumfail+=int(str(fail[key]))
Changed:
<
<
if ( sumdone >= 2000 ) :
>
>
if ( sumdone >= nfiles ) :
  lastkey=int(str(key)) break if ( key > now ) :
Line: 69 to 68
  fromsite=sys.argv[1] tosite=sys.argv[2]
Changed:
<
<
starttime=sys.argv[3] GetTransferResults? (starttime,fromsite,tosite)
>
>
nfiles=sys.argv[3] starttime=sys.argv[4] GetTransferResults? (fromsite,tosite,nfiles,starttime)
 

-- JamesLetts - 2011/05/17 \ No newline at end of file

Revision 42011/05/20 - Main.JamesLetts

Line: 1 to 1
 
META TOPICPARENT name="CMSLoadTestLHCONEResults"

Scripts

Line: 13 to 13
 import xml.dom.minidom from xml.dom.minidom import Node
Added:
>
>
 def GetTransferResults? (starttime,fromsite="T2_US_Wisconsin",tosite="T2_US_MIT",delta=86400): import urllib2, xml.dom.minidom
Changed:
<
<
url="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=300&starttime="+str(starttime)+"&endtime="+str(starttime+delta)
>
>
url="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=300&starttime="+str(starttime)+"&endtime="+str(int(starttime)+int(delta))
  xmldata = urllib2.urlopen(url).read() dom = xml.dom.minidom.parseString(xmldata)
Line: 51 to 52
  maxrate=rate

maxrate/=(1024.*1024)

Changed:
<
<
latency=float(lastkey-starttime)/3600.
>
>
latency=float(int(lastkey)-int(starttime))/3600. if latency < 0 : latency=-1 try :
  quality=float(sumdone)/float(sumdone+sumfail)*100
Added:
>
>
except ZeroDivisionError? : quality=-1
  print sumdone, "files completed"
Changed:
<
<
print("%.2f" % maxrate),"MiB/s" print("%.2f" % latency),"h" print("%.2f" % quality),"%" print "Start time:",time.ctime(starttime) print "End time: ",time.ctime(lastkey) print "Now: ",now
>
>
print("%.1f" % maxrate),"MiB/s" print("%.1f" % latency),"h" print("%.1f" % quality),"%"
  return
Changed:
<
<
starttime=1305614014 GetTransferResults? (starttime,"T2_UK_London_IC","T2_US_Purdue")
>
>
fromsite=sys.argv[1] tosite=sys.argv[2] starttime=sys.argv[3] GetTransferResults? (starttime,fromsite,tosite)
 

-- JamesLetts - 2011/05/17 \ No newline at end of file

Revision 32011/05/20 - Main.JamesLetts

Line: 1 to 1
 
META TOPICPARENT name="CMSLoadTestLHCONEResults"

Scripts

Line: 8 to 8
 #!/usr/bin/python # import sys,os.path
Changed:
<
<
import sets,string,time
>
>
import string,time
 import urllib import xml.dom.minidom from xml.dom.minidom import Node

Revision 22011/05/17 - Main.JamesLetts

Line: 1 to 1
 
META TOPICPARENT name="CMSLoadTestLHCONEResults"

Scripts

Line: 31 to 31
  sumdone=0 sumfail=0 lastkey=0
Added:
>
>
now=time.ctime()
  for key in sorted(done.iterkeys()): sumdone+=int(str(done[key])) sumfail+=int(str(fail[key]))
Changed:
<
<
if ( sumdone >= 1000 ) :
>
>
if ( sumdone >= 2000 ) :
  lastkey=int(str(key)) break
Added:
>
>
if ( key > now ) : break
  url2="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=3600&starttime="+str(starttime)+"&endtime="+str(lastkey) xmldata2 = urllib2.urlopen(url2).read()
Line: 51 to 54
  latency=float(lastkey-starttime)/3600. quality=float(sumdone)/float(sumdone+sumfail)*100
Changed:
<
<
print("%.2f" % maxrate),"MB/s"
>
>
print sumdone, "files completed" print("%.2f" % maxrate),"MiB/s"
  print("%.2f" % latency),"h" print("%.2f" % quality),"%" print "Start time:",time.ctime(starttime) print "End time: ",time.ctime(lastkey)
Changed:
<
<
print "Now: ",time.ctime()
>
>
print "Now: ",now
  return
Changed:
<
<
starttime=1305334800 GetTransferResults? (starttime)
>
>
starttime=1305614014 GetTransferResults? (starttime,"T2_UK_London_IC","T2_US_Purdue")
 

-- JamesLetts - 2011/05/17

Revision 12011/05/17 - Main.JamesLetts

Line: 1 to 1
Added:
>
>
META TOPICPARENT name="CMSLoadTestLHCONEResults"

Scripts

Script for extracting the maximum hourly transfer rate, latency and quality over a certain link.

#!/usr/bin/python
#
import sys,os.path
import sets,string,time
import urllib
import xml.dom.minidom
from xml.dom.minidom import Node

def GetTransferResults(starttime,fromsite="T2_US_Wisconsin",tosite="T2_US_MIT",delta=86400):
  import urllib2, xml.dom.minidom 
  url="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=300&starttime="+str(starttime)+"&endtime="+str(starttime+delta)
  xmldata = urllib2.urlopen(url).read() 
  dom = xml.dom.minidom.parseString(xmldata) 

  done={}
  fail={}
  for node in dom.getElementsByTagName('transfer'): 
    done_files=node.attributes["done_files"].value
    fail_files=node.attributes["fail_files"].value
    time_bin=node.attributes["timebin"].value
    done[time_bin]=done_files
    fail[time_bin]=fail_files

  sumdone=0
  sumfail=0
  lastkey=0
  for key in sorted(done.iterkeys()):
    sumdone+=int(str(done[key]))
    sumfail+=int(str(fail[key]))
    if ( sumdone >= 1000 ) :
      lastkey=int(str(key))
      break

  url2="https://cmsweb.cern.ch/phedex/datasvc/xml/debug/TransferHistory?from="+fromsite+"&to="+tosite+"&binwidth=3600&starttime="+str(starttime)+"&endtime="+str(lastkey)
  xmldata2 = urllib2.urlopen(url2).read() 
  dom2 = xml.dom.minidom.parseString(xmldata2) 
  maxrate=0.
  for node in dom2.getElementsByTagName('transfer'): 
    rate=float(str(node.attributes["rate"].value))
    if ( rate > maxrate ) :
      maxrate=rate

  maxrate/=(1024.*1024)
  latency=float(lastkey-starttime)/3600.
  quality=float(sumdone)/float(sumdone+sumfail)*100

  print("%.2f" % maxrate),"MB/s"
  print("%.2f" % latency),"h"
  print("%.2f" % quality),"%"
  print "Start time:",time.ctime(starttime)
  print "End time:  ",time.ctime(lastkey)
  print "Now:       ",time.ctime()

  return

starttime=1305334800
GetTransferResults(starttime)

-- JamesLetts - 2011/05/17

 
This site is powered by the TWiki collaboration platformCopyright © by the contributing authors. All material on this collaboration platform is the property of the contributing authors.
Ideas, requests, problems regarding TWiki? Send feedback