lib: make packets and packet messages optional, disabled by default

This patch makes it optional for the streams of a given stream class to
support the concept of packets, much like the support for discarded
events and discarded packets is already optional.

Packets exist to support trace formats which group events into packets,
all the packet's events sharing the same packet context information. CTF
is currently the single input format which needs the packet concept.
Other, more simple sources (existing and future) which do not need the
packet concept need, before this patch, to create at least one and, for
each stream, to emit the packet beginning and packet end messages at the
right locations within the message flow. This makes this part of the API
cumbersome and unnecessary.

With this patch, a message iterator can simply emit:

1. Stream beginning message
2. Event messages
3. Stream end message

API changes
===========
When you create a stream class with bt_stream_class_create() or
bt_stream_class_create_with_id(), the created stream class does NOT
support packets by default. This is to make the initial stream class's
configuration as simple as possible. This means that:

* You cannot create packet objects from the instances of this stream
  class (bt_packet_create()).

* You cannot create packet beginning and packet end messages for the
  instances of this stream class because you don't have any packet
  object (bt_message_packet_beginning_create(),
  bt_message_packet_beginning_create_with_default_clock_snapshot(),
  bt_message_packet_end_create(), and
  bt_message_packet_end_create_with_default_clock_snapshot()).

* The stream class cannot support discarded packets
  (bt_stream_class_set_supports_discarded_packets()).

* The stream class cannot have a packet context field class
  (bt_stream_class_set_packet_context_field_class()).

* You cannot create a detached packet context field from this stream
  class (bt_packet_context_field_create()).

* You cannot create an event message with a packet object
  (bt_message_event_create() and
  bt_message_event_create_with_default_clock_snapshot(); more about this
  below).

To make a stream class support packets, you need to call
bt_stream_class_set_supports_packets(). This is also where you specify
whether or not the stream class's stream packets have a beginning and/or
an end default clock snapshot from now on.

You can use bt_stream_class_supports_packets() to know if a given stream
class supports packets.

The functions bt_message_event_create() and
bt_message_event_create_with_default_clock_snapshot() are renamed to
bt_message_event_create_with_packet() and
bt_message_event_create_with_packet_and_default_clock_snapshot(). The
functions bt_message_event_create() and
bt_message_event_create_with_default_clock_snapshot() now accept a
`const bt_stream *` parameter to associate the event to a stream.

You can borrow the stream to which an event is associated with
bt_event_borrow_stream_const(), however it was created.
bt_event_borrow_packet_const() returns `NULL` if the event's stream
class does not support packets.

When a stream class supports packets, it is required that you emit the
packet beginning and packet end messages at the correct locations within
an iterator's message flow for a given instance of this stream class.

When a stream class does not support packets, it is required that you do
not create packet beginning and packet end messages for the instances of
this stream class.

Plugin changes
==============
`src.ctf.fs` and `src.ctf.lttng-live`:
    The message iterators always create stream classes supporting
    packets.

`sink.ctf.fs`:
    The component requires that packets are supported if a stream class
    also supports discarded events. This is because the way to indicate
    discarded events in CTF 1.8 is with packet contexts.

    When packets are not supported for a given stream, the component
    creates "artificial" packets. I chose to make it create packets of
    about 4 MiB. This could become configurable in the future.

`flt.lttng-utils.debug-info`:
    The message iterator copies whether or not the stream class supports
    packets.

`src.text.dmesg`:
    The message iterator does not emit packet beginning and end messages
    anymore.

`sink.text.details`:
    The component prints whether or not a given stream class supports
    packets. The component only prints the "packets have beginning
    default clock snapshot" and "packets have end default clock
    snapshot" stream class properties if packets are supported to reduce
    textual noise.

Signed-off-by: Philippe Proulx <eeppeliteloop@gmail.com>
Change-Id: I79a8063b4a85140004789d024364cf37ef076c45
Reviewed-on: https://review.lttng.org/c/babeltrace/+/1656
Reviewed-by: Simon Marchi <simon.marchi@efficios.com>
Tested-by: jenkins <jenkins@lttng.org>

include/babeltrace2/graph/message-event.h

@@ -27,7 +27,7 @@
 #include <stdint.h>
 
 /*
- * For bt_self_message_iterator, bt_event, bt_packet,
+ * For bt_self_message_iterator, bt_event, bt_packet, bt_stream,
  * bt_event_class, bt_message
  */
 #include <babeltrace2/types.h>
@@ -38,12 +38,24 @@ extern "C" {
 
 extern
 bt_message *bt_message_event_create(
+		bt_self_message_iterator *message_iterator,
+		const bt_event_class *event_class,
+		const bt_stream *stream);
+
+extern
+bt_message *bt_message_event_create_with_packet(
 		bt_self_message_iterator *message_iterator,
 		const bt_event_class *event_class,
 		const bt_packet *packet);
 
 extern
 bt_message *bt_message_event_create_with_default_clock_snapshot(
+		bt_self_message_iterator *message_iterator,
+		const bt_event_class *event_class,
+		const bt_stream *stream, uint64_t raw_clock_value);
+
+extern
+bt_message *bt_message_event_create_with_packet_and_default_clock_snapshot(
 		bt_self_message_iterator *message_iterator,
 		const bt_event_class *event_class,
 		const bt_packet *packet, uint64_t raw_clock_value);

include/babeltrace2/trace-ir/stream-class-const.h

@@ -51,6 +51,9 @@ extern bt_bool bt_stream_class_assigns_automatic_event_class_id(
 extern bt_bool bt_stream_class_assigns_automatic_stream_id(
 		const bt_stream_class *stream_class);
 
+extern bt_bool bt_stream_class_supports_packets(
+		const bt_stream_class *stream_class);
+
 extern bt_bool bt_stream_class_packets_have_beginning_default_clock_snapshot(
 		const bt_stream_class *stream_class);
 

include/babeltrace2/trace-ir/stream-class.h

@@ -67,12 +67,6 @@ extern void bt_stream_class_set_assigns_automatic_event_class_id(
 extern void bt_stream_class_set_assigns_automatic_stream_id(
 		bt_stream_class *stream_class, bt_bool value);
 
-extern void bt_stream_class_set_packets_have_beginning_default_clock_snapshot(
-		bt_stream_class *stream_class, bt_bool value);
-
-extern void bt_stream_class_set_packets_have_end_default_clock_snapshot(
-		bt_stream_class *stream_class, bt_bool value);
-
 extern void bt_stream_class_set_supports_discarded_events(
 		bt_stream_class *stream_class,
 		bt_bool supports_discarded_events,
@@ -88,6 +82,11 @@ typedef enum bt_stream_class_set_field_class_status {
 	BT_STREAM_CLASS_SET_FIELD_CLASS_STATUS_OK		= __BT_FUNC_STATUS_OK,
 } bt_stream_class_set_field_class_status;
 
+extern void bt_stream_class_set_supports_packets(
+		bt_stream_class *stream_class, bt_bool supports_packets,
+		bt_bool with_beginning_default_clock_snapshot,
+		bt_bool with_end_default_clock_snapshot);
+
 extern bt_stream_class_set_field_class_status
 bt_stream_class_set_packet_context_field_class(
 		bt_stream_class *stream_class,

src/bindings/python/bt2/bt2/__init__.py.in

@@ -47,6 +47,7 @@ from bt2.logging import *
 from bt2.message import *
 from bt2.message import _DiscardedEventsMessage
 from bt2.message import _DiscardedPacketsMessage
+from bt2.message import _EventMessage
 from bt2.message_iterator import *
 from bt2.message_iterator import _UserMessageIterator
 from bt2.packet import _Packet

src/bindings/python/bt2/bt2/event.py

@@ -48,7 +48,10 @@ def id(self):
     @property
     def packet(self):
         packet_ptr = native_bt.event_borrow_packet(self._ptr)
-        assert packet_ptr is not None
+
+        if packet_ptr is None:
+            return
+
         return bt2.packet._Packet._create_from_ptr_and_get_ref(packet_ptr)
 
     @property

src/bindings/python/bt2/bt2/message.py

@@ -60,7 +60,7 @@ class _EventMessage(_Message, _MessageWithDefaultClockSnapshot):
 
     @property
     def default_clock_snapshot(self):
-        self._check_has_default_clock_class(self.event.packet.stream.cls.default_clock_class)
+        self._check_has_default_clock_class(self.event.stream.cls.default_clock_class)
         return self._get_default_clock_snapshot(self._borrow_default_clock_snapshot_ptr)
 
     @property

src/bindings/python/bt2/bt2/message_iterator.py

@@ -165,23 +165,37 @@ def _bt_can_seek_beginning_from_native(self):
     def _bt_seek_beginning_from_native(self):
         self._seek_beginning()
 
-    def _create_event_message(self, event_class, packet, default_clock_snapshot=None):
+    def _create_event_message(self, event_class, parent=None,
+                              default_clock_snapshot=None):
         utils._check_type(event_class, bt2.event_class._EventClass)
-        utils._check_type(packet, bt2.packet._Packet)
+
+        if event_class.stream_class.supports_packets:
+            utils._check_type(parent, bt2.packet._Packet)
+        else:
+            utils._check_type(parent, bt2.stream._Stream)
 
         if default_clock_snapshot is not None:
             if event_class.stream_class.default_clock_class is None:
                 raise ValueError('event messages in this stream must not have a default clock snapshot')
 
             utils._check_uint64(default_clock_snapshot)
-            ptr = native_bt.message_event_create_with_default_clock_snapshot(
-                self._bt_ptr, event_class._ptr, packet._ptr, default_clock_snapshot)
+
+            if event_class.stream_class.supports_packets:
+                ptr = native_bt.message_event_create_with_packet_and_default_clock_snapshot(
+                    self._bt_ptr, event_class._ptr, parent._ptr, default_clock_snapshot)
+            else:
+                ptr = native_bt.message_event_create_with_default_clock_snapshot(
+                    self._bt_ptr, event_class._ptr, parent._ptr, default_clock_snapshot)
         else:
             if event_class.stream_class.default_clock_class is not None:
                 raise ValueError('event messages in this stream must have a default clock snapshot')
 
-            ptr = native_bt.message_event_create(
-                self._bt_ptr, event_class._ptr, packet._ptr)
+            if event_class.stream_class.supports_packets:
+                ptr = native_bt.message_event_create_with_packet(
+                    self._bt_ptr, event_class._ptr, parent._ptr)
+            else:
+                ptr = native_bt.message_event_create(
+                    self._bt_ptr, event_class._ptr, parent._ptr)
 
         if ptr is None:
             raise bt2.CreationError('cannot create event message object')

src/bindings/python/bt2/bt2/stream.py

@@ -52,6 +52,9 @@ def id(self):
         return id if id >= 0 else None
 
     def create_packet(self):
+        if not self.cls.supports_packets:
+            raise bt2.Error('cannot create packet: stream class does not support packets')
+
         packet_ptr = native_bt.packet_create(self._ptr)
 
         if packet_ptr is None:

src/bindings/python/bt2/bt2/stream_class.py

@@ -129,25 +129,30 @@ def _assigns_automatic_stream_id(self, auto_id):
 
     _assigns_automatic_stream_id = property(fset=_assigns_automatic_stream_id)
 
+    @property
+    def supports_packets(self):
+        return native_bt.stream_class_supports_packets(self._ptr)
+
     @property
     def packets_have_beginning_default_clock_snapshot(self):
         return native_bt.stream_class_packets_have_beginning_default_clock_snapshot(self._ptr)
 
-    def _packets_have_beginning_default_clock_snapshot(self, value):
-        utils._check_bool(value)
-        native_bt.stream_class_set_packets_have_beginning_default_clock_snapshot(self._ptr, value)
-
-    _packets_have_beginning_default_clock_snapshot = property(fset=_packets_have_beginning_default_clock_snapshot)
-
     @property
     def packets_have_end_default_clock_snapshot(self):
         return native_bt.stream_class_packets_have_end_default_clock_snapshot(self._ptr)
 
-    def _packets_have_end_default_clock_snapshot(self, value):
-        utils._check_bool(value)
-        native_bt.stream_class_set_packets_have_end_default_clock_snapshot(self._ptr, value)
+    def _set_supports_packets(self, supports, with_begin_cs=False, with_end_cs=False):
+        utils._check_bool(supports)
+        utils._check_bool(with_begin_cs)
+        utils._check_bool(with_end_cs)
+
+        if not supports and (with_begin_cs or with_end_cs):
+            raise ValueError('cannot not support packets, but have default clock snapshots')
+
+        if not supports and self.packet_context_field_class is not None:
+            raise ValueError('stream class already has a packet context field class')
 
-    _packets_have_end_default_clock_snapshot = property(fset=_packets_have_end_default_clock_snapshot)
+        native_bt.stream_class_set_supports_packets(self._ptr, supports, with_begin_cs, with_end_cs)
 
     @property
     def supports_discarded_events(self):
@@ -174,6 +179,9 @@ def _set_supports_discarded_packets(self, supports, with_cs):
         utils._check_bool(supports)
         utils._check_bool(with_cs)
 
+        if supports and not self.supports_packets:
+            raise ValueError('cannot support discarded packets, but not support packets')
+
         if not supports and with_cs:
             raise ValueError('cannot not support discarded packets, but have default clock snapshots')
 
@@ -211,6 +219,10 @@ def _packet_context_field_class(self, packet_context_field_class):
         if packet_context_field_class is not None:
             utils._check_type(packet_context_field_class,
                               bt2.field_class._StructureFieldClass)
+
+            if not self.supports_packets:
+                raise ValueError('stream class does not support packets')
+
             status = native_bt.stream_class_set_packet_context_field_class(self._ptr,
                                                                            packet_context_field_class._ptr)
             utils._handle_func_status(status,

src/bindings/python/bt2/bt2/trace_class.py

@@ -115,6 +115,7 @@ def create_stream_class(self, id=None,
                             default_clock_class=None,
                             assigns_automatic_event_class_id=True,
                             assigns_automatic_stream_id=True,
+                            supports_packets=False,
                             packets_have_beginning_default_clock_snapshot=False,
                             packets_have_end_default_clock_snapshot=False,
                             supports_discarded_events=False,
@@ -139,19 +140,28 @@ def create_stream_class(self, id=None,
         if name is not None:
             sc._name = name
 
-        if packet_context_field_class is not None:
-            sc._packet_context_field_class = packet_context_field_class
-
         if event_common_context_field_class is not None:
             sc._event_common_context_field_class = event_common_context_field_class
 
         if default_clock_class is not None:
             sc._default_clock_class = default_clock_class
 
+        # call after `sc._default_clock_class` because, if
+        # `packets_have_beginning_default_clock_snapshot` or
+        # `packets_have_end_default_clock_snapshot` is true, then this
+        # stream class needs a default clock class already.
+        sc._set_supports_packets(supports_packets,
+                                 packets_have_beginning_default_clock_snapshot,
+                                 packets_have_end_default_clock_snapshot)
+
+        # call after sc._set_supports_packets() because, if
+        # `packet_context_field_class` is not `None`, then this stream
+        # class needs to support packets already.
+        if packet_context_field_class is not None:
+            sc._packet_context_field_class = packet_context_field_class
+
         sc._assigns_automatic_event_class_id = assigns_automatic_event_class_id
         sc._assigns_automatic_stream_id = assigns_automatic_stream_id
-        sc._packets_have_beginning_default_clock_snapshot = packets_have_beginning_default_clock_snapshot
-        sc._packets_have_end_default_clock_snapshot = packets_have_end_default_clock_snapshot
         sc._set_supports_discarded_events(supports_discarded_events,
                                           discarded_events_have_default_clock_snapshots)
         sc._set_supports_discarded_packets(supports_discarded_packets,

src/lib/graph/message/event.c

@@ -82,7 +82,8 @@ static inline
 struct bt_message *create_event_message(
 		struct bt_self_message_iterator *self_msg_iter,
 		const struct bt_event_class *c_event_class,
-		const struct bt_packet *c_packet, bool with_cs,
+		const struct bt_packet *c_packet,
+		const struct bt_stream *c_stream, bool with_cs,
 		uint64_t raw_value)
 {
 	struct bt_self_component_port_input_message_iterator *msg_iter =
@@ -91,11 +92,12 @@ struct bt_message *create_event_message(
 	struct bt_event_class *event_class = (void *) c_event_class;
 	struct bt_stream_class *stream_class;
 	struct bt_packet *packet = (void *) c_packet;
+	struct bt_stream *stream = (void *) c_stream;
 	struct bt_event *event;
 
+	BT_ASSERT(stream);
 	BT_ASSERT_PRE_NON_NULL(msg_iter, "Message iterator");
 	BT_ASSERT_PRE_NON_NULL(event_class, "Event class");
-	BT_ASSERT_PRE_NON_NULL(packet, "Packet");
 	BT_ASSERT_PRE(event_class_has_trace(event_class),
 		"Event class is not part of a trace: %!+E", event_class);
 	stream_class = bt_event_class_borrow_stream_class_inline(event_class);
@@ -109,7 +111,7 @@ struct bt_message *create_event_message(
 		"cs-val=%" PRIu64,
 		event_class, stream_class, with_cs, raw_value);
 	BT_LIB_LOGD("Creating event message object: %![ec-]+E", event_class);
-	event = bt_event_create(event_class, packet);
+	event = bt_event_create(event_class, packet, stream);
 	if (G_UNLIKELY(!event)) {
 		BT_LIB_LOGE_APPEND_CAUSE(
 			"Cannot create event from event class: "
@@ -152,7 +154,12 @@ struct bt_message *create_event_message(
 
 	BT_ASSERT(!message->event);
 	message->event = event;
-	bt_packet_set_is_frozen(packet, true);
+
+	if (packet) {
+		bt_packet_set_is_frozen(packet, true);
+	}
+
+	bt_stream_freeze(stream);
 	bt_event_class_freeze(event_class);
 	BT_LIB_LOGD("Created event message object: "
 		"%![msg-]+n, %![event-]+e", message, event);
@@ -167,21 +174,45 @@ struct bt_message *create_event_message(
 }
 
 struct bt_message *bt_message_event_create(
+		struct bt_self_message_iterator *msg_iter,
+		const struct bt_event_class *event_class,
+		const struct bt_stream *stream)
+{
+	BT_ASSERT_PRE_NON_NULL(stream, "Stream");
+	return create_event_message(msg_iter, event_class, NULL, stream, false, 0);
+}
+
+struct bt_message *bt_message_event_create_with_packet(
 		struct bt_self_message_iterator *msg_iter,
 		const struct bt_event_class *event_class,
 		const struct bt_packet *packet)
 {
-	return create_event_message(msg_iter, event_class, packet, false, 0);
+	BT_ASSERT_PRE_NON_NULL(packet, "Packet");
+	return create_event_message(msg_iter, event_class, packet,
+		packet->stream, false, 0);
 }
 
 struct bt_message *bt_message_event_create_with_default_clock_snapshot(
+		struct bt_self_message_iterator *msg_iter,
+		const struct bt_event_class *event_class,
+		const struct bt_stream *stream,
+		uint64_t raw_value)
+{
+	BT_ASSERT_PRE_NON_NULL(stream, "Stream");
+	return create_event_message(msg_iter, event_class, NULL, stream,
+		true, raw_value);
+}
+
+struct bt_message *
+bt_message_event_create_with_packet_and_default_clock_snapshot(
 		struct bt_self_message_iterator *msg_iter,
 		const struct bt_event_class *event_class,
 		const struct bt_packet *packet,
 		uint64_t raw_value)
 {
+	BT_ASSERT_PRE_NON_NULL(packet, "Packet");
 	return create_event_message(msg_iter, event_class, packet,
-		true, raw_value);
+		packet->stream, true, raw_value);
 }
 
 BT_HIDDEN

src/lib/graph/message/packet.c

@@ -99,6 +99,12 @@ struct bt_message *create_packet_message(
 	stream_class = bt_stream_borrow_class(stream);
 	BT_ASSERT(stream_class);
 
+	/*
+	 * It's not possible to create a packet from a stream of which
+	 * the class indicates that packets are not supported.
+	 */
+	BT_ASSERT(stream_class->supports_packets);
+
 	if (pool == &msg_iter->graph->packet_begin_msg_pool) {
 		need_cs = stream_class->packets_have_beginning_default_clock_snapshot;
 	} else {