headers
Dan Stromberg | 1 Jun 2010 06:48
Picon

[Cython] Exceptions?


I'm attempting to get a Cython module to raise exceptions that'll be visible to the calling CPython.

The Cython code in question looks like:
   def add_from_fileno(self, fileno, length_to_add):
      exception_string = self.add_from_fileno_c(fileno, length_to_add)
      if exception_string.startswith("Buffer error"):
         raise exceptions.BufferError, exception_string
      elif exception_string == '':
         pass
      else:
         raise exceptions.AssertionError, exception_string

   cdef add_from_fileno_c(self, fileno, length_to_add):
      if self.would_overflow(length_to_add):
         return "Buffer error: Would overflow"
      # FIXME: We need to do something about EINTR on these 3 read's!
      if self.would_add_wrap(length_to_add):
         # do the read in two pieces
         first_length, second_length = self.split_to_two_add(length_to_add)
         length_added = read(fileno, &self.buffer[self.end], first_length)
         if length_added != first_length:
            return "length_added != first_length"
         length_added = read(fileno, self.buffer, second_length)
         if length_added != second_length:
            return "length_added != second_length"
         self.end = second_length
      else:
         # do the read in one piece
         length_added = read(fileno, &self.buffer[self.end], length_to_add)
         if length_added != length_to_add:
            return "length_added != length_to_add"
         self.end += length_to_add
      return ''
...and in the caller (CPython code, inheriting from unittest):
   def test_fileno_from_file_overflow(self):
      file_ = open('input-file', 'w')
      file_.write('abc' * 15)
      file_.close()

      rb = ring_buffer_mod.Ring_buffer(buffer_size = 10)
      input_fileno = os.open('input-file', os.O_RDONLY)
      for i in xrange(10):
         rb.add_from_fileno(input_fileno, 1)
      #self.assertRaises(exceptions.BufferError, rb.add_from_fileno, input_fileno, 1)
      try:
         rb.add_from_fileno(input_fileno, 1)
      except exceptions.BufferError:
         pass

All seems fine, except for the exceptions.  When I raise an exception in Cython, I see a message on my terminal (I believe there should be none), and the calling CPython code doesn't appear to realize that an exception has been raised.  I added the add_from_fileno_c/add_from_file distinction, because I was hoping that a def would be able to raise an exception, after finding that cdef's and cpdef's seemed to have problems with it - but it appears that there's some sort of exception barrier between Cython and CPython.

I've googled quite a bit, but haven't found much on the topic that didn't seem kind of hand wavey.

What do I need to do, to raise an exception in Cython, that CPython code will be able to see?

Thanks!

<div>
<br>
I'm attempting to get a Cython module to raise exceptions that'll be
visible to the calling CPython.<br><br>
The Cython code in question looks like:<br><blockquote>&nbsp;&nbsp; def add_from_fileno(self, fileno, length_to_add):<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; exception_string = self.add_from_fileno_c(fileno, length_to_add)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; if exception_string.startswith("Buffer error"):<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; raise exceptions.BufferError, exception_string<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; elif exception_string == '':<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; pass<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; else:<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; raise exceptions.AssertionError, exception_string<br><br>
&nbsp;&nbsp; cdef add_from_fileno_c(self, fileno, length_to_add):<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; if self.would_overflow(length_to_add):<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; return "Buffer error: Would overflow"<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; # FIXME: We need to do something about EINTR on these 3 read's!<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; if self.would_add_wrap(length_to_add):<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; # do the read in two pieces<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; first_length, second_length =
self.split_to_two_add(length_to_add)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; length_added = read(fileno, &amp;self.buffer[self.end],
first_length)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; if length_added != first_length:<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; return "length_added != first_length"<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; length_added = read(fileno, self.buffer, second_length)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; if length_added != second_length:<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; return "length_added != second_length"<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; self.end = second_length<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; else:<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; # do the read in one piece<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; length_added = read(fileno, &amp;self.buffer[self.end],
length_to_add)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; if length_added != length_to_add:<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; return "length_added != length_to_add"<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; self.end += length_to_add<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; return ''<br>
</blockquote>
...and in the caller (CPython code, inheriting from unittest):<br><blockquote>&nbsp;&nbsp; def test_fileno_from_file_overflow(self):<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; file_ = open('input-file', 'w')<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; file_.write('abc' * 15)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; file_.close()<br><br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; rb = ring_buffer_mod.Ring_buffer(buffer_size = 10)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; input_fileno = os.open('input-file', os.O_RDONLY)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; for i in xrange(10):<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; rb.add_from_fileno(input_fileno, 1)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; #self.assertRaises(exceptions.BufferError, rb.add_from_fileno,
input_fileno, 1)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; try:<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; rb.add_from_fileno(input_fileno, 1)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; except exceptions.BufferError:<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; pass<br>
</blockquote>
<br>
All seems fine, except for the exceptions.&nbsp; When I raise an exception
in Cython, I see a message on my terminal (I believe there should be
none), and the calling CPython code doesn't appear to realize that an
exception has been raised.&nbsp; I added the add_from_fileno_c/add_from_file
distinction, because I was hoping that a def would be able to raise an
exception, after finding that cdef's and cpdef's seemed to have
problems with it - but it appears that there's some sort of exception
barrier between Cython and CPython.<br><br>
I've googled quite a bit, but haven't found much on the topic that
didn't seem kind of hand wavey.<br><br>
What do I need to do, to raise an exception in Cython, that CPython
code will be able to see?<br><br>
Thanks!<br><br>
</div>
Permalink | Reply | 4 comments |
headers
Robert Bradshaw | 1 Jun 2010 07:21
Favicon

Re: [Cython] Exceptions?

On May 31, 2010, at 9:48 PM, Dan Stromberg wrote:

>
> I'm attempting to get a Cython module to raise exceptions that'll be  
> visible to the calling CPython.
>
> The Cython code in question looks like:
>    def add_from_fileno(self, fileno, length_to_add):
>       exception_string = self.add_from_fileno_c(fileno, length_to_add)
>       if exception_string.startswith("Buffer error"):
>          raise exceptions.BufferError, exception_string
>       elif exception_string == '':
>          pass
>       else:
>          raise exceptions.AssertionError, exception_string
>
>    cdef add_from_fileno_c(self, fileno, length_to_add):
>       if self.would_overflow(length_to_add):
>          return "Buffer error: Would overflow"
>       # FIXME: We need to do something about EINTR on these 3 read's!
>       if self.would_add_wrap(length_to_add):
>          # do the read in two pieces
>          first_length, second_length =  
> self.split_to_two_add(length_to_add)
>          length_added = read(fileno, &self.buffer[self.end],  
> first_length)
>          if length_added != first_length:
>             return "length_added != first_length"
>          length_added = read(fileno, self.buffer, second_length)
>          if length_added != second_length:
>             return "length_added != second_length"
>          self.end = second_length
>       else:
>          # do the read in one piece
>          length_added = read(fileno, &self.buffer[self.end],  
> length_to_add)
>          if length_added != length_to_add:
>             return "length_added != length_to_add"
>          self.end += length_to_add
>       return ''
> ...and in the caller (CPython code, inheriting from unittest):
>    def test_fileno_from_file_overflow(self):
>       file_ = open('input-file', 'w')
>       file_.write('abc' * 15)
>       file_.close()
>
>       rb = ring_buffer_mod.Ring_buffer(buffer_size = 10)
>       input_fileno = os.open('input-file', os.O_RDONLY)
>       for i in xrange(10):
>          rb.add_from_fileno(input_fileno, 1)
>       #self.assertRaises(exceptions.BufferError, rb.add_from_fileno,  
> input_fileno, 1)
>       try:
>          rb.add_from_fileno(input_fileno, 1)
>       except exceptions.BufferError:
>          pass
>
> All seems fine, except for the exceptions.  When I raise an  
> exception in Cython, I see a message on my terminal (I believe there  
> should be none), and the calling CPython code doesn't appear to  
> realize that an exception has been raised.  I added the  
> add_from_fileno_c/add_from_file distinction, because I was hoping  
> that a def would be able to raise an exception, after finding that  
> cdef's and cpdef's seemed to have problems with it - but it appears  
> that there's some sort of exception barrier between Cython and  
> CPython.
>
> I've googled quite a bit, but haven't found much on the topic that  
> didn't seem kind of hand wavey.
>
> What do I need to do, to raise an exception in Cython, that CPython  
> code will be able to see?

This should work just fine for def functions, as well as c(p)def  
functions not declaring a c return type (in which case you should see  
[1]). Have you tried calling this directly from the command line  
(eliminating the possibility that it's something with the unittest  
framework?) What if you do

cdef class A:
     def spam(self):
         raise TypeError
     cpdef eggs(self):
         raise ValueError

Does that work for you? (It does for me.)

- Robert

[1] http://docs.cython.org/src/userguide/language_basics.html#error-return-values
Permalink | Reply | 2 comments |
headers
Stefan Behnel | 1 Jun 2010 08:12
Picon
Favicon

Re: [Cython] Exceptions?

Dan Stromberg, 01.06.2010 06:48:
> All seems fine, except for the exceptions. When I raise an exception in
> Cython, I see a message on my terminal (I believe there should be none),

I wonder why you didn't think of presenting that "message on your terminal" 
here, so that we know what it says and can /tell/ you if there really 
"should be none".

> I added the add_from_fileno_c/add_from_file
> distinction, because I was hoping that a def would be able to raise an
> exception, after finding that cdef's and cpdef's seemed to have problems
> with it - but it appears that there's some sort of exception barrier
> between Cython and CPython.

No, there isn't. There may be one in your code if you use cdef functions, 
as those can only propagate exceptions when a) their return type implicitly 
allows it or b) they are explicitly declared to raise an exception. You 
should read the Cython docs on exceptions.

Also note that this question would have been more appropriate for the 
cython-users mailing list.

Stefan
Permalink | Reply | 0 comments |
headers
Stefan Behnel | 1 Jun 2010 13:30
Picon
Favicon

[Cython] Hudson jobs for Haoyu's GSoC

Hi,

I set up the Hudson jobs for the gsoc branch:

https://sage.math.washington.edu:8091/hudson/view/cython-haoyu

Happy coding,

Stefan
Permalink | Reply | 0 comments |
headers
Dan Stromberg | 2 Jun 2010 05:43
Picon

Re: [Cython] Exceptions?

On 05/31/2010 10:21 PM, Robert Bradshaw wrote:
> On May 31, 2010, at 9:48 PM, Dan Stromberg wrote:
>
>    
>> I'm attempting to get a Cython module to raise exceptions that'll be
>> visible to the calling CPython.
>>
>> The Cython code in question looks like:
>>     def add_from_fileno(self, fileno, length_to_add):
>>        exception_string = self.add_from_fileno_c(fileno, length_to_add)
>>        if exception_string.startswith("Buffer error"):
>>           raise exceptions.BufferError, exception_string
>>        elif exception_string == '':
>>           pass
>>        else:
>>           raise exceptions.AssertionError, exception_string
>>
>>     cdef add_from_fileno_c(self, fileno, length_to_add):
>>        if self.would_overflow(length_to_add):
>>           return "Buffer error: Would overflow"
>>        # FIXME: We need to do something about EINTR on these 3 read's!
>>        if self.would_add_wrap(length_to_add):
>>           # do the read in two pieces
>>           first_length, second_length =
>> self.split_to_two_add(length_to_add)
>>           length_added = read(fileno,&self.buffer[self.end],
>> first_length)
>>           if length_added != first_length:
>>              return "length_added != first_length"
>>           length_added = read(fileno, self.buffer, second_length)
>>           if length_added != second_length:
>>              return "length_added != second_length"
>>           self.end = second_length
>>        else:
>>           # do the read in one piece
>>           length_added = read(fileno,&self.buffer[self.end],
>> length_to_add)
>>           if length_added != length_to_add:
>>              return "length_added != length_to_add"
>>           self.end += length_to_add
>>        return ''
>> ...and in the caller (CPython code, inheriting from unittest):
>>     def test_fileno_from_file_overflow(self):
>>        file_ = open('input-file', 'w')
>>        file_.write('abc' * 15)
>>        file_.close()
>>
>>        rb = ring_buffer_mod.Ring_buffer(buffer_size = 10)
>>        input_fileno = os.open('input-file', os.O_RDONLY)
>>        for i in xrange(10):
>>           rb.add_from_fileno(input_fileno, 1)
>>        #self.assertRaises(exceptions.BufferError, rb.add_from_fileno,
>> input_fileno, 1)
>>        try:
>>           rb.add_from_fileno(input_fileno, 1)
>>        except exceptions.BufferError:
>>           pass
>>
>> All seems fine, except for the exceptions.  When I raise an
>> exception in Cython, I see a message on my terminal (I believe there
>> should be none), and the calling CPython code doesn't appear to
>> realize that an exception has been raised.  I added the
>> add_from_fileno_c/add_from_file distinction, because I was hoping
>> that a def would be able to raise an exception, after finding that
>> cdef's and cpdef's seemed to have problems with it - but it appears
>> that there's some sort of exception barrier between Cython and
>> CPython.
>>
>> I've googled quite a bit, but haven't found much on the topic that
>> didn't seem kind of hand wavey.
>>
>> What do I need to do, to raise an exception in Cython, that CPython
>> code will be able to see?
>>      
> This should work just fine for def functions, as well as c(p)def
> functions not declaring a c return type (in which case you should see
> [1]). Have you tried calling this directly from the command line
> (eliminating the possibility that it's something with the unittest
> framework?) What if you do
>
> cdef class A:
>       def spam(self):
>           raise TypeError
>       cpdef eggs(self):
>           raise ValueError
>
> Does that work for you? (It does for me.)
>
> - Robert
>
> [1] http://docs.cython.org/src/userguide/language_basics.html#error-return-values
>
> _______________________________________________
> Cython-dev mailing list
> Cython-dev@...
> http://codespeak.net/mailman/listinfo/cython-dev
>    
You're right - it must be something about the unittest module - for some 
reason, it's printing the exception I'm trying to catch.  It works as 
expected interactively.

Apologies for not going to the cython-users list - I wasn't aware of it 
when I sent my message, and had subscribed to cython-dev (only) long ago.

Thanks for the cool software and assistance!

PS: I might suggest adding something to the FAQ about cdef functions not 
declaring a return type doing this well.  It's kind of there, but kind 
of not - and in a FAQ, it's perhaps best to have it fully there.
Permalink | Reply | 1 comment |
headers
Dag Sverre Seljebotn | 2 Jun 2010 08:34
Picon
Picon

Re: [Cython] Exceptions?

Dan Stromberg wrote:
> On 05/31/2010 10:21 PM, Robert Bradshaw wrote:
>   
>> On May 31, 2010, at 9:48 PM, Dan Stromberg wrote:
>>
>>    
>>     
>>> I'm attempting to get a Cython module to raise exceptions that'll be
>>> visible to the calling CPython.
>>>
>>> The Cython code in question looks like:
>>>     def add_from_fileno(self, fileno, length_to_add):
>>>        exception_string = self.add_from_fileno_c(fileno, length_to_add)
>>>        if exception_string.startswith("Buffer error"):
>>>           raise exceptions.BufferError, exception_string
>>>        elif exception_string == '':
>>>           pass
>>>        else:
>>>           raise exceptions.AssertionError, exception_string
>>>
>>>     cdef add_from_fileno_c(self, fileno, length_to_add):
>>>        if self.would_overflow(length_to_add):
>>>           return "Buffer error: Would overflow"
>>>        # FIXME: We need to do something about EINTR on these 3 read's!
>>>        if self.would_add_wrap(length_to_add):
>>>           # do the read in two pieces
>>>           first_length, second_length =
>>> self.split_to_two_add(length_to_add)
>>>           length_added = read(fileno,&self.buffer[self.end],
>>> first_length)
>>>           if length_added != first_length:
>>>              return "length_added != first_length"
>>>           length_added = read(fileno, self.buffer, second_length)
>>>           if length_added != second_length:
>>>              return "length_added != second_length"
>>>           self.end = second_length
>>>        else:
>>>           # do the read in one piece
>>>           length_added = read(fileno,&self.buffer[self.end],
>>> length_to_add)
>>>           if length_added != length_to_add:
>>>              return "length_added != length_to_add"
>>>           self.end += length_to_add
>>>        return ''
>>> ...and in the caller (CPython code, inheriting from unittest):
>>>     def test_fileno_from_file_overflow(self):
>>>        file_ = open('input-file', 'w')
>>>        file_.write('abc' * 15)
>>>        file_.close()
>>>
>>>        rb = ring_buffer_mod.Ring_buffer(buffer_size = 10)
>>>        input_fileno = os.open('input-file', os.O_RDONLY)
>>>        for i in xrange(10):
>>>           rb.add_from_fileno(input_fileno, 1)
>>>        #self.assertRaises(exceptions.BufferError, rb.add_from_fileno,
>>> input_fileno, 1)
>>>        try:
>>>           rb.add_from_fileno(input_fileno, 1)
>>>        except exceptions.BufferError:
>>>           pass
>>>
>>> All seems fine, except for the exceptions.  When I raise an
>>> exception in Cython, I see a message on my terminal (I believe there
>>> should be none), and the calling CPython code doesn't appear to
>>> realize that an exception has been raised.  I added the
>>> add_from_fileno_c/add_from_file distinction, because I was hoping
>>> that a def would be able to raise an exception, after finding that
>>> cdef's and cpdef's seemed to have problems with it - but it appears
>>> that there's some sort of exception barrier between Cython and
>>> CPython.
>>>
>>> I've googled quite a bit, but haven't found much on the topic that
>>> didn't seem kind of hand wavey.
>>>
>>> What do I need to do, to raise an exception in Cython, that CPython
>>> code will be able to see?
>>>      
>>>       
>> This should work just fine for def functions, as well as c(p)def
>> functions not declaring a c return type (in which case you should see
>> [1]). Have you tried calling this directly from the command line
>> (eliminating the possibility that it's something with the unittest
>> framework?) What if you do
>>
>> cdef class A:
>>       def spam(self):
>>           raise TypeError
>>       cpdef eggs(self):
>>           raise ValueError
>>
>> Does that work for you? (It does for me.)
>>
>> - Robert
>>
>> [1] http://docs.cython.org/src/userguide/language_basics.html#error-return-values
>>
>> _______________________________________________
>> Cython-dev mailing list
>> Cython-dev@...
>> http://codespeak.net/mailman/listinfo/cython-dev
>>    
>>     
> You're right - it must be something about the unittest module - for some 
> reason, it's printing the exception I'm trying to catch.  It works as 
> expected interactively.
>
> Apologies for not going to the cython-users list - I wasn't aware of it 
> when I sent my message, and had subscribed to cython-dev (only) long ago.
>
> Thanks for the cool software and assistance!
>
> PS: I might suggest adding something to the FAQ about cdef functions not 
> declaring a return type doing this well.  It's kind of there, but kind 
> of not - and in a FAQ, it's perhaps best to have it fully there.
>   
Please go ahead!

Dag
Permalink | Reply | 0 comments |
headers
Chuck Blake | 8 Jun 2010 15:23
Picon
Picon

[Cython] line-directives support for Distutils build_ext

Hey, guys.  I did this a long time ago, but it patches cleanly against the
cython-devel tip (at least yesterday's tip).

There was some discussion of what the default should be for the #line/#file
directives Robert added.  This is a hopefully non-contentious addition to
support pushing that decision to higher levels of the build system.

All this patch does is add pyrex_line_directives to support that.  It's
an analogue of the extra attributes .pyrex_include_dirs going with -I,
or .pyrex_cplus going with the --cplus command option, etc.

This allows you to add .pyrex_line_directives to Extension() objects in
your setup.py.  The appropriate command option is then added to the
invocations of cython for those extensions, as it is for the -I/--cplus/..
options.

--------------------------------------------------------------------------
diff -r dd29215de4a0 Cython/Distutils/build_ext.py
--- a/Cython/Distutils/build_ext.py	Mon Jun 07 11:06:45 2010 -0700
+++ b/Cython/Distutils/build_ext.py	Mon Jun 07 16:45:43 2010 -0400
 <at>  <at>  -36,6 +36,8  <at>  <at> 
          "generate C++ source files"),
         ('pyrex-create-listing', None,
          "write errors to a listing file"),
+        ('pyrex-line-directives', None,
+         "emit source line directives"),
         ('pyrex-include-dirs=', None,
          "path to the Cython include files" + sep_by),
         ('pyrex-c-in-temp', None,
 <at>  <at>  -45,13 +47,14  <at>  <at> 
         ])

     boolean_options.extend([
-        'pyrex-cplus', 'pyrex-create-listing', 'pyrex-c-in-temp'
+        'pyrex-cplus', 'pyrex-create-listing', 'pyrex-line-directives', 'pyrex-c-in-temp'
     ])

     def initialize_options(self):
         _build_ext.build_ext.initialize_options(self)
         self.pyrex_cplus = 0
         self.pyrex_create_listing = 0
+        self.pyrex_line_directives = 0
         self.pyrex_include_dirs = None
         self.pyrex_c_in_temp = 0
         self.pyrex_gen_pxi = 0
 <at>  <at>  -114,6 +117,8  <at>  <at> 

         create_listing = self.pyrex_create_listing or \
             getattr(extension, 'pyrex_create_listing', 0)
+        line_directives = self.pyrex_line_directives or \
+            getattr(extension, 'pyrex_line_directives', 0)
         cplus = self.pyrex_cplus or getattr(extension, 'pyrex_cplus', 0) or \
                 (extension.language and extension.language.lower() == 'c++')
         pyrex_gen_pxi = self.pyrex_gen_pxi or getattr(extension, 'pyrex_gen_pxi', 0)
 <at>  <at>  -186,6 +191,7  <at>  <at> 
                     include_path = includes,
                     output_file = target,
                     cplus = cplus,
+                    emit_linenums = line_directives,
                     generate_pxi = pyrex_gen_pxi)
                 result = cython_compile(source, options=options,
                                         full_module_name=module_name)
diff -r dd29215de4a0 Cython/Distutils/extension.py
--- a/Cython/Distutils/extension.py	Mon Jun 07 11:06:45 2010 -0700
+++ b/Cython/Distutils/extension.py	Mon Jun 07 16:45:43 2010 -0400
 <at>  <at>  -21,6 +21,8  <at>  <at> 
         Unix form for portability)
     pyrex_create_listing_file : boolean
         write pyrex error messages to a listing (.lis) file.
+    pyrex_line_directivess : boolean
+        emit pyx line numbers for debugging/profiling
     pyrex_cplus : boolean
         use the C++ compiler for compiling and linking.
     pyrex_c_in_temp : boolean
 <at>  <at>  -70,6 +72,7  <at>  <at> 

         self.pyrex_include_dirs = pyrex_include_dirs or []
         self.pyrex_create_listing = pyrex_create_listing
+        self.pyrex_line_directives = pyrex_line_directives
         self.pyrex_cplus = pyrex_cplus
         self.pyrex_c_in_temp = pyrex_c_in_temp
         self.pyrex_gen_pxi = pyrex_gen_pxi
Permalink | Reply | 5 comments |
headers
Chuck Blake | 8 Jun 2010 19:57
Picon
Picon

[Cython] Buffer API Code Generation Bug

Say you want to implement an extension in Cython exporting a Buffer API.
This is a great canonical use-case for cython.  So, you write something
(stripped to its bare essentials) like

    cdef class Foo:
        def __getreadbuffer__ (Foo f, long s, void **A): pass
        def __getwritebuffer__(Foo f, long s, void **A): pass
        def __getsegcount__   (Foo f, int *C):
            if C: C[0] = 1
            return 1
        def __getbuffer__(Foo f, Py_buffer *b, int flags): pass

Currently, Cython generates a PyBufferProcs initializer like

    static PyBufferProcs __pyx_tp_as_buffer_Foo = {
        #if PY_MAJOR_VERSION < 3
        __pyx_pf_3int_3Foo___getreadbuffer__, /*bf_getreadbuffer*/
        #endif

with appropriate method definitions like

    static int __pyx_pf_3int_3Foo___getsegcount__(PyObject *__pyx_v_f, int *__pyx_v_C) {

That is fine for Python 2.3/2.4 and Python 3.0.

However, for Python 2.5 and 2.6, object.h changed the type of the function
arguments for segment counts (and a lot of other things) to Py_ssize_t
which can be 64-bits.

For the read/write/charbuffer methods this isn't that big a deal if you
have less than 4 giga-segments since the arguments are values and they
should be converted, though you do get a C compiler warning.

However, for the [get]segcountproc method (the typedef name dropped the
"get" for 2.5 and added it back in for 2.6), it does actually matter to
get the type right.  Otherwise, Cython defines a function taking an int*
but calls are made through a function pointer the C compiler thinks is
an Py_ssize_t*.  So, assuming the call site really uses the address of
a Py_ssize_t variable, the generated assembly from the assignment through
the int* will only alter 4 of the 8 bytes at the region passed to it.

On little-endian 64-bit architectures, it still works out all right.
On big-endian 64-bit architectures (e.g. PPC/IBM's POWER), it will
erroneously convert some small number of segments like "1" into some
crazy giant number of giga-segments (which will likely translate into
some kind of segmentation violation downstream).  So, it's a rare and
kind of subtle bug.  Anyway, one reason to pay attention to certain
C compiler warnings for the generated code.

So, we really want to instead generate a signature like

  static Py_ssize_t __pyx_pf_7ssize_t_3Foo___getsegcount__(PyObject *__pyx_v_f, Py_ssize_t
*__pyx_v_C) {

The following patch does exactly this.  Note that the generated code is
correct for all Python 2,3 versions because of Cython generating typedefs
of Py_ssize_t to "int" for backward compatibility.  So for those versions
the Py_ssize_t will be effectively int like it is in the current generation
interface, though it will be spelled differently in emitted code.

We also fix the signatures for other PyBufferProcs entries to be complete.

The only real issue I had was that it may err too much on the side of
backward compatibility within Cython itself instead of being more
internally consistent.  The Z/z choice reverses the format_map convention
of "more capitialization = more indirection" as used by p/P, i/I, s/S.
Capital Z was already taken.

On the other hand, Z is only used by about 6 signatures that I see
anywhere in all of Cython and format_map itself seems isolated.
So, it is cleaner to go with the more consistent approach and flip
the case of the Z in those 6 sigs and make z=Py_ssize_t and Z=Py_ssize_t*.

If you want, I'm happy to prepare a patch that does z/Z the way i/I are.
It's pretty trivial text editing, though, to search for Signature.*Z and
make the change in TypeSlots.py.  For what it's worth, I've tested this
patch and the code it generates on Py 2.4, 2.5, 2.6, and 3.1 and it
seems fine.  3.0 was harder as my Gentoo isn't currently letting me
install that simultaneously with 3.1, but I don't think there is any
issue.

Thanks!

--------------------------------------------------------------------------
diff -r dd29215de4a0 Cython/Compiler/TypeSlots.py
--- a/Cython/Compiler/TypeSlots.py	Mon Jun 07 11:06:45 2010 -0700
+++ b/Cython/Compiler/TypeSlots.py	Mon Jun 07 16:59:49 2010 -0400
 <at>  <at>  -49,6 +49,7  <at>  <at> 
         'I': PyrexTypes.c_int_ptr_type,
         'l': PyrexTypes.c_long_type,
         'Z': PyrexTypes.c_py_ssize_t_type,
+        'z': PyrexTypes.c_py_ssize_t_ptr_type, # Z here z above would be more conventional but invasive
         's': PyrexTypes.c_char_ptr_type,
         'S': PyrexTypes.c_char_ptr_ptr_type,
         'r': PyrexTypes.c_returncode_type,
 <at>  <at>  -506,7 +507,7  <at>  <at> 
 getcharbufferproc = Signature("TiS", 'i')  # typedef int (*getcharbufferproc)(PyObject *, int, const
char **);
 readbufferproc = Signature("TZP", "Z")     # typedef Py_ssize_t (*readbufferproc)(PyObject *,
Py_ssize_t, void **);
 writebufferproc = Signature("TZP", "Z")    # typedef Py_ssize_t (*writebufferproc)(PyObject *,
Py_ssize_t, void **);
-segcountproc = Signature("TZ", "Z")        # typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
+segcountproc = Signature("Tz", "Z")        # typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
 charbufferproc = Signature("TZS", "Z")     # typedef Py_ssize_t (*charbufferproc)(PyObject *,
Py_ssize_t, char **);
 objargproc = Signature("TO", 'r')          # typedef int (*objobjproc)(PyObject *, PyObject *);
                                            # typedef int (*visitproc)(PyObject *, void *);
 <at>  <at>  -625,10 +626,10  <at>  <at> 
 )

 PyBufferProcs = (
-    MethodSlot(getreadbufferproc, "bf_getreadbuffer", "__getreadbuffer__", py3 = False),
-    MethodSlot(getwritebufferproc, "bf_getwritebuffer", "__getwritebuffer__", py3 = False),
-    MethodSlot(getsegcountproc, "bf_getsegcount", "__getsegcount__", py3 = False),
-    MethodSlot(getcharbufferproc, "bf_getcharbuffer", "__getcharbuffer__", py3 = False),
+    MethodSlot(readbufferproc, "bf_getreadbuffer", "__getreadbuffer__", py3 = False),
+    MethodSlot(writebufferproc, "bf_getwritebuffer", "__getwritebuffer__", py3 = False),
+    MethodSlot(segcountproc, "bf_getsegcount", "__getsegcount__", py3 = False),
+    MethodSlot(charbufferproc, "bf_getcharbuffer", "__getcharbuffer__", py3 = False),

     MethodSlot(getbufferproc, "bf_getbuffer", "__getbuffer__", ifdef = "PY_VERSION_HEX >= 0x02060000"),
     MethodSlot(releasebufferproc, "bf_releasebuffer", "__releasebuffer__", ifdef = "PY_VERSION_HEX
>= 0x02060000")
Permalink | Reply | 4 comments |
headers
Lisandro Dalcin | 9 Jun 2010 02:54
Picon
Gravatar

Re: [Cython] Buffer API Code Generation Bug

On 8 June 2010 14:57, Chuck Blake <cb <at> pdos.csail.mit.edu> wrote:
> Say you want to implement an extension in Cython exporting a Buffer API.
> This is a great canonical use-case for cython.  So, you write something
> (stripped to its bare essentials) like
>
>    cdef class Foo:
>        def __getreadbuffer__ (Foo f, long s, void **A): pass
>        def __getwritebuffer__(Foo f, long s, void **A): pass
>        def __getsegcount__   (Foo f, int *C):
>            if C: C[0] = 1
>            return 1
>        def __getbuffer__(Foo f, Py_buffer *b, int flags): pass
>
> Currently, Cython generates a PyBufferProcs initializer like
>
>    static PyBufferProcs __pyx_tp_as_buffer_Foo = {
>        #if PY_MAJOR_VERSION < 3
>        __pyx_pf_3int_3Foo___getreadbuffer__, /*bf_getreadbuffer*/
>        #endif
>
> with appropriate method definitions like
>
>    static int __pyx_pf_3int_3Foo___getsegcount__(PyObject *__pyx_v_f, int *__pyx_v_C) {
>
> That is fine for Python 2.3/2.4 and Python 3.0.
>
>
> However, for Python 2.5 and 2.6, object.h changed the type of the function
> arguments for segment counts (and a lot of other things) to Py_ssize_t
> which can be 64-bits.
>
> For the read/write/charbuffer methods this isn't that big a deal if you
> have less than 4 giga-segments since the arguments are values and they
> should be converted, though you do get a C compiler warning.
>
> However, for the [get]segcountproc method (the typedef name dropped the
> "get" for 2.5 and added it back in for 2.6), it does actually matter to
> get the type right.  Otherwise, Cython defines a function taking an int*
> but calls are made through a function pointer the C compiler thinks is
> an Py_ssize_t*.  So, assuming the call site really uses the address of
> a Py_ssize_t variable, the generated assembly from the assignment through
> the int* will only alter 4 of the 8 bytes at the region passed to it.
>
> On little-endian 64-bit architectures, it still works out all right.
> On big-endian 64-bit architectures (e.g. PPC/IBM's POWER), it will
> erroneously convert some small number of segments like "1" into some
> crazy giant number of giga-segments (which will likely translate into
> some kind of segmentation violation downstream).  So, it's a rare and
> kind of subtle bug.  Anyway, one reason to pay attention to certain
> C compiler warnings for the generated code.
>
> So, we really want to instead generate a signature like
>
>  static Py_ssize_t __pyx_pf_7ssize_t_3Foo___getsegcount__(PyObject *__pyx_v_f, Py_ssize_t
*__pyx_v_C) {
>
>
> The following patch does exactly this.  Note that the generated code is
> correct for all Python 2,3 versions because of Cython generating typedefs
> of Py_ssize_t to "int" for backward compatibility.  So for those versions
> the Py_ssize_t will be effectively int like it is in the current generation
> interface, though it will be spelled differently in emitted code.
>
> We also fix the signatures for other PyBufferProcs entries to be complete.
>
> The only real issue I had was that it may err too much on the side of
> backward compatibility within Cython itself instead of being more
> internally consistent.  The Z/z choice reverses the format_map convention
> of "more capitialization = more indirection" as used by p/P, i/I, s/S.
> Capital Z was already taken.
>
> On the other hand, Z is only used by about 6 signatures that I see
> anywhere in all of Cython and format_map itself seems isolated.
> So, it is cleaner to go with the more consistent approach and flip
> the case of the Z in those 6 sigs and make z=Py_ssize_t and Z=Py_ssize_t*.
>
> If you want, I'm happy to prepare a patch that does z/Z the way i/I are.
> It's pretty trivial text editing, though, to search for Signature.*Z and
> make the change in TypeSlots.py.  For what it's worth, I've tested this
> patch and the code it generates on Py 2.4, 2.5, 2.6, and 3.1 and it
> seems fine.  3.0 was harder as my Gentoo isn't currently letting me
> install that simultaneously with 3.1, but I don't think there is any
> issue.
>
> Thanks!
>
> --------------------------------------------------------------------------
> diff -r dd29215de4a0 Cython/Compiler/TypeSlots.py
> --- a/Cython/Compiler/TypeSlots.py      Mon Jun 07 11:06:45 2010 -0700
> +++ b/Cython/Compiler/TypeSlots.py      Mon Jun 07 16:59:49 2010 -0400
>  <at>  <at>  -49,6 +49,7  <at>  <at> 
>         'I': PyrexTypes.c_int_ptr_type,
>         'l': PyrexTypes.c_long_type,
>         'Z': PyrexTypes.c_py_ssize_t_type,
> +        'z': PyrexTypes.c_py_ssize_t_ptr_type, # Z here z above would be more conventional but invasive
>         's': PyrexTypes.c_char_ptr_type,
>         'S': PyrexTypes.c_char_ptr_ptr_type,
>         'r': PyrexTypes.c_returncode_type,
>  <at>  <at>  -506,7 +507,7  <at>  <at> 
>  getcharbufferproc = Signature("TiS", 'i')  # typedef int (*getcharbufferproc)(PyObject *, int,
const char **);
>  readbufferproc = Signature("TZP", "Z")     # typedef Py_ssize_t (*readbufferproc)(PyObject *,
Py_ssize_t, void **);
>  writebufferproc = Signature("TZP", "Z")    # typedef Py_ssize_t (*writebufferproc)(PyObject *,
Py_ssize_t, void **);
> -segcountproc = Signature("TZ", "Z")        # typedef Py_ssize_t (*segcountproc)(PyObject *,
Py_ssize_t *);
> +segcountproc = Signature("Tz", "Z")        # typedef Py_ssize_t (*segcountproc)(PyObject *,
Py_ssize_t *);
>  charbufferproc = Signature("TZS", "Z")     # typedef Py_ssize_t (*charbufferproc)(PyObject *,
Py_ssize_t, char **);
>  objargproc = Signature("TO", 'r')          # typedef int (*objobjproc)(PyObject *, PyObject *);
>                                            # typedef int (*visitproc)(PyObject *, void *);
>  <at>  <at>  -625,10 +626,10  <at>  <at> 
>  )
>
>  PyBufferProcs = (
> -    MethodSlot(getreadbufferproc, "bf_getreadbuffer", "__getreadbuffer__", py3 = False),
> -    MethodSlot(getwritebufferproc, "bf_getwritebuffer", "__getwritebuffer__", py3 = False),
> -    MethodSlot(getsegcountproc, "bf_getsegcount", "__getsegcount__", py3 = False),
> -    MethodSlot(getcharbufferproc, "bf_getcharbuffer", "__getcharbuffer__", py3 = False),
> +    MethodSlot(readbufferproc, "bf_getreadbuffer", "__getreadbuffer__", py3 = False),
> +    MethodSlot(writebufferproc, "bf_getwritebuffer", "__getwritebuffer__", py3 = False),
> +    MethodSlot(segcountproc, "bf_getsegcount", "__getsegcount__", py3 = False),
> +    MethodSlot(charbufferproc, "bf_getcharbuffer", "__getcharbuffer__", py3 = False),
>
>     MethodSlot(getbufferproc, "bf_getbuffer", "__getbuffer__", ifdef = "PY_VERSION_HEX >= 0x02060000"),
>     MethodSlot(releasebufferproc, "bf_releasebuffer", "__releasebuffer__", ifdef =
"PY_VERSION_HEX >= 0x02060000")
> _______________________________________________
> Cython-dev mailing list
> Cython-dev <at> codespeak.net
> http://codespeak.net/mailman/listinfo/cython-dev
>

Looks fine, but I think you should follow the i/I rules. Any other opinion?

--

-- 
Lisandro Dalcin
---------------
CIMEC (INTEC/CONICET-UNL)
Predio CONICET-Santa Fe
Colectora RN 168 Km 472, Paraje El Pozo
Tel: +54-342-4511594 (ext 1011)
Tel/Fax: +54-342-4511169
_______________________________________________
Cython-dev mailing list
Cython-dev <at> codespeak.net
http://codespeak.net/mailman/listinfo/cython-dev
Permalink | Reply | 0 comments |
headers
Lisandro Dalcin | 9 Jun 2010 02:56
Picon
Gravatar

Re: [Cython] line-directives support for Distutils build_ext

On 8 June 2010 10:23, Chuck Blake <cb <at> pdos.csail.mit.edu> wrote:
> Hey, guys.  I did this a long time ago, but it patches cleanly against the
> cython-devel tip (at least yesterday's tip).
>
> There was some discussion of what the default should be for the #line/#file
> directives Robert added.  This is a hopefully non-contentious addition to
> support pushing that decision to higher levels of the build system.
>
> All this patch does is add pyrex_line_directives to support that.  It's
> an analogue of the extra attributes .pyrex_include_dirs going with -I,
> or .pyrex_cplus going with the --cplus command option, etc.
>
> This allows you to add .pyrex_line_directives to Extension() objects in
> your setup.py.  The appropriate command option is then added to the
> invocations of cython for those extensions, as it is for the -I/--cplus/..
> options.
>
> --------------------------------------------------------------------------
> diff -r dd29215de4a0 Cython/Distutils/build_ext.py
> --- a/Cython/Distutils/build_ext.py     Mon Jun 07 11:06:45 2010 -0700
> +++ b/Cython/Distutils/build_ext.py     Mon Jun 07 16:45:43 2010 -0400
>  <at>  <at>  -36,6 +36,8  <at>  <at> 
>          "generate C++ source files"),
>         ('pyrex-create-listing', None,
>          "write errors to a listing file"),
> +        ('pyrex-line-directives', None,
> +         "emit source line directives"),
>         ('pyrex-include-dirs=', None,
>          "path to the Cython include files" + sep_by),
>         ('pyrex-c-in-temp', None,
>  <at>  <at>  -45,13 +47,14  <at>  <at> 
>         ])
>
>     boolean_options.extend([
> -        'pyrex-cplus', 'pyrex-create-listing', 'pyrex-c-in-temp'
> +        'pyrex-cplus', 'pyrex-create-listing', 'pyrex-line-directives', 'pyrex-c-in-temp'
>     ])
>
>     def initialize_options(self):
>         _build_ext.build_ext.initialize_options(self)
>         self.pyrex_cplus = 0
>         self.pyrex_create_listing = 0
> +        self.pyrex_line_directives = 0
>         self.pyrex_include_dirs = None
>         self.pyrex_c_in_temp = 0
>         self.pyrex_gen_pxi = 0
>  <at>  <at>  -114,6 +117,8  <at>  <at> 
>
>         create_listing = self.pyrex_create_listing or \
>             getattr(extension, 'pyrex_create_listing', 0)
> +        line_directives = self.pyrex_line_directives or \
> +            getattr(extension, 'pyrex_line_directives', 0)
>         cplus = self.pyrex_cplus or getattr(extension, 'pyrex_cplus', 0) or \
>                 (extension.language and extension.language.lower() == 'c++')
>         pyrex_gen_pxi = self.pyrex_gen_pxi or getattr(extension, 'pyrex_gen_pxi', 0)
>  <at>  <at>  -186,6 +191,7  <at>  <at> 
>                     include_path = includes,
>                     output_file = target,
>                     cplus = cplus,
> +                    emit_linenums = line_directives,
>                     generate_pxi = pyrex_gen_pxi)
>                 result = cython_compile(source, options=options,
>                                         full_module_name=module_name)
> diff -r dd29215de4a0 Cython/Distutils/extension.py
> --- a/Cython/Distutils/extension.py     Mon Jun 07 11:06:45 2010 -0700
> +++ b/Cython/Distutils/extension.py     Mon Jun 07 16:45:43 2010 -0400
>  <at>  <at>  -21,6 +21,8  <at>  <at> 
>         Unix form for portability)
>     pyrex_create_listing_file : boolean
>         write pyrex error messages to a listing (.lis) file.
> +    pyrex_line_directivess : boolean
> +        emit pyx line numbers for debugging/profiling
>     pyrex_cplus : boolean
>         use the C++ compiler for compiling and linking.
>     pyrex_c_in_temp : boolean
>  <at>  <at>  -70,6 +72,7  <at>  <at> 
>
>         self.pyrex_include_dirs = pyrex_include_dirs or []
>         self.pyrex_create_listing = pyrex_create_listing
> +        self.pyrex_line_directives = pyrex_line_directives
>         self.pyrex_cplus = pyrex_cplus
>         self.pyrex_c_in_temp = pyrex_c_in_temp
>         self.pyrex_gen_pxi = pyrex_gen_pxi
> _______________________________________________
> Cython-dev mailing list
> Cython-dev <at> codespeak.net
> http://codespeak.net/mailman/listinfo/cython-dev
>

Looks fine. Could you generate the patch with hg export and send it attached?

--

-- 
Lisandro Dalcin
---------------
CIMEC (INTEC/CONICET-UNL)
Predio CONICET-Santa Fe
Colectora RN 168 Km 472, Paraje El Pozo
Tel: +54-342-4511594 (ext 1011)
Tel/Fax: +54-342-4511169
_______________________________________________
Cython-dev mailing list
Cython-dev <at> codespeak.net
http://codespeak.net/mailman/listinfo/cython-dev
Permalink | Reply | 4 comments |

Gmane