@@ -268,15 +268,12 @@ def apply( self ):
268
268
print '%s DEBUG: Querying dataset \' %s\' with' % ( self ._label , dataset )
269
269
print ' \' %s\' ' % ( dasQuery )
270
270
# partially stolen from das_client.py for option '--format=plain', needs filter ("grep") in the query
271
- dasData = das_client .get_data ( 'https://cmsweb.cern.ch' , dasQuery , 0 , dasLimit , False )
272
- jsondict = json .loads ( dasData )
271
+ jsondict = das_client .get_data ( 'https://cmsweb.cern.ch' , dasQuery , 0 , dasLimit , False )
273
272
if debug :
274
- print '%s DEBUG: Received DAS data:' % ( self ._label )
275
- print ' \' %s\' ' % ( dasData )
276
- print '%s DEBUG: Determined JSON dictionary:' % ( self ._label )
273
+ print '%s DEBUG: Received DAS JSON dictionary:' % ( self ._label )
277
274
print ' \' %s\' ' % ( jsondict )
278
275
if jsondict [ 'status' ] != 'ok' :
279
- print 'There was a problem while querying DAS with query \' %s\' . Server reply was:\n %s' % (dasQuery , dasData )
276
+ print 'There was a problem while querying DAS with query \' %s\' . Server reply was:\n %s' % (dasQuery , jsondict )
280
277
exit ( 1 )
281
278
mongo_query = jsondict [ 'mongo_query' ]
282
279
filters = mongo_query [ 'filters' ]
@@ -289,20 +286,17 @@ def apply( self ):
289
286
print '%s DEBUG: Data in JSON dictionary:' % ( self ._label )
290
287
print ' \' %s\' ' % ( data )
291
288
for row in data :
292
- filePath = [ r for r in das_client .get_value ( row , filters ) ][ 0 ]
289
+ filePath = [ r for r in das_client .get_value ( row , filters [ 'grep' ] ) ][ 0 ]
293
290
if debug :
294
291
print '%s DEBUG: Testing file entry \' %s\' ' % ( self ._label , filePath )
295
292
if len ( filePath ) > 0 :
296
293
if validVersion != version :
297
- dasTest = das_client .get_data ( 'https://cmsweb.cern.ch' , 'site dataset=%s | grep site.name' % ( dataset ), 0 , 999 , False )
298
- jsontestdict = json .loads ( dasTest )
294
+ jsontestdict = das_client .get_data ( 'https://cmsweb.cern.ch' , 'site dataset=%s | grep site.name' % ( dataset ), 0 , 999 , False )
299
295
mongo_testquery = jsontestdict [ 'mongo_query' ]
300
296
testfilters = mongo_testquery [ 'filters' ]
301
297
testdata = jsontestdict [ 'data' ]
302
298
if debug :
303
- print '%s DEBUG: Received DAS data (site test):' % ( self ._label )
304
- print ' \' %s\' ' % ( dasTest )
305
- print '%s DEBUG: Determined JSON dictionary (site test):' % ( self ._label )
299
+ print '%s DEBUG: Received DAS JSON dictionary (site test):' % ( self ._label )
306
300
print ' \' %s\' ' % ( jsontestdict )
307
301
print '%s DEBUG: Query in JSON dictionary (site test):' % ( self ._label )
308
302
print ' \' %s\' ' % ( mongo_testquery )
@@ -312,7 +306,7 @@ def apply( self ):
312
306
print ' \' %s\' ' % ( testdata )
313
307
foundSE = False
314
308
for testrow in testdata :
315
- siteName = [ tr for tr in das_client .get_value ( testrow , testfilters ) ][ 0 ]
309
+ siteName = [ tr for tr in das_client .get_value ( testrow , testfilters [ 'grep' ] ) ][ 0 ]
316
310
if siteName == domainSE :
317
311
foundSE = True
318
312
break
@@ -343,57 +337,58 @@ def apply( self ):
343
337
else :
344
338
if debug :
345
339
print '%s DEBUG: Using DBS query' % ( self ._label )
346
- for version in range ( maxVersions , 0 , - 1 ):
347
- filePaths = []
348
- fileCount = 0
349
- dataset = '/%s/%s-%s-v%i/%s' % ( relVal , cmsswVersion , globalTag , version , dataTier )
350
- dbsQuery = 'find file where dataset = %s' % ( dataset )
351
- if debug :
352
- print '%s DEBUG: Querying dataset \' %s\' with' % ( self ._label , dataset )
353
- print ' \' %s\' ' % ( dbsQuery )
354
- foundSE = False
355
- for line in os .popen ( 'dbs search --query="%s"' % ( dbsQuery ) ):
356
- if line .find ( '.root' ) != - 1 :
357
- if validVersion != version :
358
- if not foundSE :
359
- dbsSiteQuery = 'find dataset where dataset = %s and site = %s' % ( dataset , domainSE )
360
- if debug :
361
- print '%s DEBUG: Querying site \' %s\' with' % ( self ._label , domainSE )
362
- print ' \' %s\' ' % ( dbsSiteQuery )
363
- for lineSite in os .popen ( 'dbs search --query="%s"' % ( dbsSiteQuery ) ):
364
- if lineSite .find ( dataset ) != - 1 :
365
- foundSE = True
366
- break
367
- if not foundSE :
368
- if debug :
369
- print '%s DEBUG: Possible version \' v%s\' not available on SE \' %s\' ' % ( self ._label , version , domainSE )
370
- break
371
- validVersion = version
372
- if debug :
373
- print '%s DEBUG: Valid version set to \' v%i\' ' % ( self ._label , validVersion )
374
- if numberOfFiles == 0 :
375
- break
376
- filePath = line .replace ( '\n ' , '' )
377
- if debug :
378
- print '%s DEBUG: File \' %s\' found' % ( self ._label , filePath )
379
- fileCount += 1
380
- if fileCount > skipFiles :
381
- filePaths .append ( filePath )
382
- if not numberOfFiles < 0 :
383
- if numberOfFiles <= len ( filePaths ):
384
- break
385
- if validVersion > 0 :
386
- if numberOfFiles == 0 and debug :
387
- print '%s DEBUG: No files requested' % ( self ._label )
388
- break
340
+ print '%s WARNING: DBS query disabled for DBS3 transition to new API' % ( self ._label )
341
+ #for version in range( maxVersions, 0, -1 ):
342
+ #filePaths = []
343
+ #fileCount = 0
344
+ #dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
345
+ #dbsQuery = 'find file where dataset = %s'%( dataset )
346
+ #if debug:
347
+ #print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
348
+ #print ' \'%s\''%( dbsQuery )
349
+ #foundSE = False
350
+ #for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ).readlines():
351
+ #if line.find( '.root' ) != -1:
352
+ #if validVersion != version:
353
+ #if not foundSE:
354
+ #dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
355
+ #if debug:
356
+ #print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
357
+ #print ' \'%s\''%( dbsSiteQuery )
358
+ #for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ).readlines():
359
+ #if lineSite.find( dataset ) != -1:
360
+ #foundSE = True
361
+ #break
362
+ #if not foundSE:
363
+ #if debug:
364
+ #print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
365
+ #break
366
+ #validVersion = version
367
+ #if debug:
368
+ #print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
369
+ #if numberOfFiles == 0:
370
+ #break
371
+ #filePath = line.replace( '\n', '' )
372
+ #if debug:
373
+ #print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
374
+ #fileCount += 1
375
+ #if fileCount > skipFiles:
376
+ #filePaths.append( filePath )
377
+ #if not numberOfFiles < 0:
378
+ #if numberOfFiles <= len( filePaths ):
379
+ #break
380
+ #if validVersion > 0:
381
+ #if numberOfFiles == 0 and debug:
382
+ #print '%s DEBUG: No files requested'%( self._label )
383
+ #break
389
384
390
385
# Check output and return
391
386
if validVersion == 0 :
392
- print '%s INFO : No RelVal file(s) found at all in datasets \' %s*\' on SE \' %s\' ' % ( self ._label , datasetAll , domainSE )
387
+ print '%s WARNING : No RelVal file(s) found at all in datasets \' %s*\' on SE \' %s\' ' % ( self ._label , datasetAll , domainSE )
393
388
if debug :
394
389
self .messageEmptyList ()
395
390
elif len ( filePaths ) == 0 :
396
- print '%s INFO : No RelVal file(s) picked up in dataset \' %s\' ' % ( self ._label , dataset )
391
+ print '%s WARNING : No RelVal file(s) picked up in dataset \' %s\' ' % ( self ._label , dataset )
397
392
if debug :
398
393
self .messageEmptyList ()
399
394
elif len ( filePaths ) < numberOfFiles :
0 commit comments