WIP: implement stream in cwrap (Scheduled for 2.2)

python/python/cwrap/CMakeLists.txt

@@ -4,6 +4,7 @@ set(PYTHON_SOURCES
     basecenum.py
     basecvalue.py
     cfile.py
+    stream.py
     clib.py
     metacwrap.py
     prototype.py

python/python/cwrap/__init__.py

@@ -44,9 +44,10 @@
 
 from .cfile import CFILE
 from .clib import load, lib_name
+from .stream import Stream
 
 from .metacwrap import MetaCWrap
 from .prototype import REGISTERED_TYPES, Prototype, PrototypeError
 
 __all__ = ['BaseCClass', 'BaseCEnum', 'BaseCValue', 'CFILE',
-           'MetaCWrap', 'Prototype', 'load', 'lib_name']
+           'MetaCWrap', 'Prototype', 'load', 'lib_name', 'Stream']

python/python/cwrap/stream.py

@@ -0,0 +1,182 @@
+#  Copyright (C) 2017  Statoil ASA, Norway.
+#
+#  This file is part of ERT - Ensemble based Reservoir Tool.
+#
+#  ERT is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ERT is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.
+#
+#  See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
+#  for more details.
+
+from os import linesep
+from six import string_types
+
+import numpy
+from .clib import load as cwrapload
+from .prototype import Prototype
+from .basecclass import BaseCClass
+
+
+class LibcPrototype(Prototype):
+    lib = cwrapload('libc', '.6')
+
+    def __init__(self, prototype, bind=False):
+        super(LibcPrototype, self).__init__(LibcPrototype.lib, prototype, bind=bind)
+
+class Stream(BaseCClass):
+    """
+    Utility class to map a Python file handle <-> FILE* in C
+    """
+    TYPE_NAME = "stream"
+
+    _fopen  = LibcPrototype("void* fopen(char*, char*)")
+    _fread  = LibcPrototype("size_t   fread(void*, size_t, size_t, stream)")
+    _fwrite = LibcPrototype("size_t   fwrite(void*, size_t, size_t, stream)")
+    _fseek  = LibcPrototype("size_t   fseek(stream, size_t, size_t)", bind=True)
+
+    _fclose = LibcPrototype("size_t   fclose (stream)", bind=True)
+    _fflush = LibcPrototype("size_t   fflush (stream)", bind=True)
+
+    def __init__(self, fname, mode='r'):
+        c_ptr = self._fopen(fname, mode)
+        self._mode = mode
+        self._fname = fname
+
+        self._closed = False # A closed file cannot be used for further I/O
+                             # operations.  close() may be called more than once
+                             # without error.
+
+        try:
+            super(Stream, self).__init__(c_ptr)
+        except ValueError as e:
+            self._closed = True
+            raise IOError('Could not load file "%s" in mode %s.' % (fname, mode))
+
+    @property
+    def closed(self):
+        return self._closed
+
+    def _assert_open(self):
+        if self.closed:
+            raise IOError('Stream is closed: %s' % self)
+
+    def _read_rest_of_file(self):
+        out = numpy.zeros(0, dtype=numpy.byte)
+        read = -1
+        while read != 0: # num bytes read
+            bptr = numpy.zeros(1, dtype=numpy.byte)
+            read = self._fread(bptr.ctypes.data, 1, 1, self)
+            if read:
+                out = numpy.append(out, bptr)
+        return out
+
+    def read(self, size=-1):
+        """Will read size bytes, or rest of file if size < 0.
+
+        The return type will be iterable, but may be a string or a numpy bytes
+        array.
+
+        If the file is open in textmode (no 'b'), it will return a string.
+        """
+        self._assert_open()
+        ret_arr = None
+        if size < 0:
+            ret_arr = self._read_rest_of_file()
+        else:
+            byte_array = numpy.zeros(size, dtype=numpy.byte)
+            self._fread(byte_array.ctypes.data, 1, size, self)
+            ret_arr = byte_array
+        if self._textmode():
+            return ret_arr.tostring()
+        return ret_arr
+
+    def _is_newline_chr(self, c, sep=None):
+        if sep is None:
+            return c == ord(linesep)
+        return c == ord(sep)
+
+    def readline(self, sep=None):
+        """ returns string.  sep defaults to linesep=\n """
+        self._assert_open()
+        if not self._textmode():
+            raise IOError('Warning: reading line from %s in byte mode is nonsensical.' % self)
+        out = numpy.zeros(0, dtype=numpy.byte)
+        while True:
+            bptr = numpy.zeros(1, dtype=numpy.byte)
+            read = self._fread(bptr.ctypes.data, 1, 1, self)
+            if read == 0:
+                break # EOF
+            out = numpy.append(out, bptr)
+            if self._is_newline_chr(bptr[0], sep):
+                break
+        return out.tostring()
+
+    def readlines(self, sep=None):
+        return [x for x in self.readline(sep=sep)]
+
+    def __iter__(self):
+        self._assert_open()
+        prev = self.readline()
+        while len(prev) > 0:
+            yield prev
+            prev = self.readline()
+
+    def seek(self, offset, whence):
+        """whence must be one of SEEK_SET, SEEK_CUR, or SEEK_END."""
+        self._assert_open()
+        if self._fseek(offset, whence):
+            raise IOError("Unable to seek.")
+
+    def _bytemode(self):
+        return 'b' in self._mode
+
+    def _textmode(self):
+        return 'b' not in self._mode
+
+    def _writable(self):
+        return any(m in self._mode for m in 'aw+')
+
+    def write(self, np_arr):
+        """ @type np_arr: numpy.array """
+        if not self._writable():
+            raise IOError('Cannot write to file "%s" in mode %s.' % (self._fname, self._mode))
+        if self.closed:
+            raise IOError('File is closed.  Cannot write to file "%s".' % self._fname)
+
+        if isinstance(np_arr, string_types):
+            arr = numpy.zeros(len(np_arr), dtype=numpy.character)
+            for i in range(len(np_arr)):
+                arr[i] = np_arr[i]
+            np_arr = arr
+        return self._fwrite(np_arr.ctypes.data, 1, len(np_arr), self)
+
+    def close(self):
+        if not self.closed:
+            self._fflush()
+            cs = self._fclose()
+            self._closed = True
+            return cs
+
+    def __repr__(self):
+        cl = ', closed' if self.closed else ''
+        fmt = 'fname=%s, mode=%s%s'
+        return self._create_repr(fmt % (self._fname, self._mode, cl))
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+        return exc_type is None
+
+    def free(self):
+        self.close()
+
+    def __del__(self):
+        self.close()

python/python/ert/ecl/ecl_file.py

@@ -202,7 +202,7 @@ def __init__( self , filename , flags = 0):
         FIPNUM from an INIT file.
         """
         c_ptr = self._open( filename , flags )
-        if c_ptr is None:
+        if not c_ptr:
             raise IOError('Failed to open file "%s"' % filename)
         else:
             super(EclFile , self).__init__(c_ptr)

python/python/ert/ecl/ecl_grid.py

@@ -23,13 +23,16 @@
 implemented in the EclGrid class. The ecl_grid module is a thin
 wrapper around the ecl_grid.c implementation from the libecl library.
 """
+from __future__ import absolute_import
+
 import ctypes
 
 import numpy
 import sys
 import os.path
 import math
-from cwrap import CFILE, BaseCClass
+from cwrap import BaseCClass
+from cwrap import Stream
 from ert.util import IntVector
 from ert.ecl import EclPrototype, EclDataType, EclKW, FortIO, EclUnitTypeEnum
 
@@ -125,24 +128,25 @@ def loadFromGrdecl(cls , filename):
         faster.
         """
 
-        if os.path.isfile(filename):
-            with open(filename) as f:
-                specgrid = EclKW.read_grdecl(f, "SPECGRID", ecl_type=EclDataType.ECL_INT, strict=False)
-                zcorn = EclKW.read_grdecl(f, "ZCORN")
-                coord = EclKW.read_grdecl(f, "COORD")
-                try:
-                    actnum = EclKW.read_grdecl(f, "ACTNUM", ecl_type=EclDataType.ECL_INT)
-                except ValueError:
-                    actnum = None
-
-                try:
-                    mapaxes = EclKW.read_grdecl(f, "MAPAXES")
-                except ValueError:
-                    mapaxes = None
-
-            return EclGrid.create( specgrid , zcorn , coord , actnum , mapaxes )
-        else:
-            raise IOError("No such file:%s" % filename)
+        if not os.path.isfile(filename):
+            raise IOError('No such file "%s".' % filename)
+
+        specgrid , zcorn , coord , actnum , mapaxes  = None, None, None, None, None
+        with Stream(filename) as f:
+            specgrid = EclKW.read_grdecl(f, "SPECGRID", ecl_type=EclTypeEnum.ECL_INT_TYPE, strict=False)
+            zcorn = EclKW.read_grdecl(f, "ZCORN")
+            coord = EclKW.read_grdecl(f, "COORD")
+            try:
+                actnum = EclKW.read_grdecl(f, "ACTNUM", ecl_type=EclTypeEnum.ECL_INT_TYPE)
+            except ValueError:
+                actnum = None
+            try:
+                mapaxes = EclKW.read_grdecl(f, "MAPAXES")
+            except ValueError:
+                mapaxes = None
+        if None in (specgrid, zcorn, coord):
+            raise ValueError('None of (specgrid, zcorn, coord) can be None: was (%s, %s, %s).' % (specgrid,zcorn,coord))
+        return EclGrid.create( specgrid , zcorn , coord , actnum , mapaxes )
 
     @classmethod
     def loadFromFile(cls , filename):
@@ -223,7 +227,6 @@ def __init__(self , filename , apply_mapaxes = True):
             super(EclGrid, self).__init__(c_ptr)
         else:
             raise IOError("Loading grid from:%s failed" % filename)
-        self.__str__ = self.__repr__
 
     def free(self):
         self._free( )

python/python/ert/ecl/ecl_kw.py

@@ -43,8 +43,9 @@
 import ctypes
 import warnings
 
-import  numpy
-from cwrap import CFILE, BaseCClass
+import numpy
+from cwrap import BaseCClass
+from cwrap import Stream
 from ert.ecl import EclDataType
 from ert.ecl import EclTypeEnum, EclUtil, EclPrototype
 
@@ -274,10 +275,12 @@ def read_grdecl( cls , fileH , kw , strict = True , ecl_type = None):
         it finds in the file.
         """
 
-        cfile  = CFILE( fileH )
-        if kw:
-            if len(kw) > 8:
-                raise TypeError("Sorry keyword:%s is too long, must be eight characters or less." % kw)
+        if not isinstance(fileH, Stream):
+            raise ValueError('Must be a Stream, fileH was a %s.' % type(fileH))
+        if kw and len(kw) > 8:
+            raise TypeError('Sorry keyword "%s" is too long, ' +
+                            'must be eight characters or less, was %d.'
+                            % (kw, len(kw)))
 
         if ecl_type is None:
             if cls.int_kw_set.__contains__( kw ):
@@ -321,8 +324,8 @@ def fseek_grdecl( cls , fileH , kw , rewind = False):
         true the function rewind to the beginning of the file and
         search from there after the initial search.
         """
-        cfile = CFILE( fileH )
-        return cls._fseek_grdecl( kw , rewind , cfile)
+        # cfile = CFILE( fileH )
+        return cls._fseek_grdecl( kw , rewind , fileH)
 
 
     @classmethod
@@ -817,14 +820,18 @@ def equal(self , other):
         The check is based on the content of the keywords, and not
         pointer comparison.
         """
-        if isinstance(other , EclKW):
+        if isinstance(other, EclKW):
             return self._equal( other )
         else:
             raise TypeError("Can only compare with another EclKW")
 
 
     def __eq__(self , other):
-        return self.equal( other )
+        if other is None:
+            return False
+        if isinstance(other, EclKW):
+            return self.equal(other)
+        return NotImplemented
 
     def __hash__(self):
         return hash(self._get_header( ))
@@ -1049,7 +1056,7 @@ def numpyCopy(self):
     def fwrite( self , fortio ):
         self._fwrite( fortio )
 
-    def write_grdecl( self , file ):
+    def write_grdecl( self , fileH ):
         """
         Will write keyword in GRDECL format.
 
@@ -1071,12 +1078,12 @@ def write_grdecl( self , file ):
             fileH.close()
 
         """
-        cfile = CFILE( file )
-        self._fprintf_grdecl( cfile )
+        # cfile = CFILE( file )
+        self._fprintf_grdecl( fileH )
 
 
 
-    def fprintf_data( self , file , fmt = None):
+    def fprintf_data( self , fileH , fmt = None):
         """
         Will print the keyword data formatted to file.
 
@@ -1094,8 +1101,8 @@ def fprintf_data( self , file , fmt = None):
         """
         if fmt is None:
             fmt = self.str_fmt + "\n"
-        cfile = CFILE( file )
-        self._fprintf_data( fmt , cfile )
+        #cfile = CFILE( file )
+        self._fprintf_data( fmt , fileH )
 
 
     def fixUninitialized(self , grid):

python/python/ert/ecl/ecl_rft.py

@@ -69,7 +69,10 @@ class EclRFT(BaseCClass):
 
     def __init__(self , name , type_string , date , days):
         c_ptr = self._alloc( name , type_string , CTime( date ) , days )
-        super(EclRFT , self).__init__( c_ptr )
+        if c_ptr:
+            super(EclRFT , self).__init__( c_ptr )
+        else:
+            raise ValueError('Failed to construct EclRFT object from given input.')
 
 
     def free(self):