Merge pull request 'fibers' (dart-lang#11) from fibers into main

Reviewed-on: http://git.local/dependencies/dart/pulls/11

runtime/vm/compiler/backend/il.cc

@@ -8567,37 +8567,29 @@ void CoroutineInitializeInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
 #if defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64)
   SPILLS_LR_TO_FRAME({});
 #endif
-  __ EnterFrame(0);
   __ PushObject(compiler::NullObject());
   __ PushRegister(kCoroutine);
   __ CallRuntime(kEnterCoroutineRuntimeEntry, 1);
   __ PopRegister(kCoroutine);
   __ Drop(1);
-  __ LeaveFrame();
-
-  __ set_constant_pool_allowed(false);
-  __ EnterDartFrame(0);
 
-  __ LoadFieldFromOffset(SPREG, kCoroutine, Coroutine::stack_base_offset());
   __ PushRegister(FPREG);
+  __ StoreFieldToOffset(SPREG, kCoroutine, Coroutine::root_stack_base_offset());
 
+  __ LoadFieldFromOffset(SPREG, kCoroutine, Coroutine::stack_base_offset());
   __ PushRegister(kCoroutine);
-  compiler->EmitCallToStub(StubCode::CoroutineEntry());
+  compiler->GenerateStubCall(source(), StubCode::CoroutineEntry(), UntaggedPcDescriptors::kOther, locs(), deopt_id(), env());
   __ PopRegister(kCoroutine);
+  __ StoreFieldToOffset(SPREG, kCoroutine, Coroutine::stack_base_offset());
 
+  __ LoadFieldFromOffset(SPREG, kCoroutine, Coroutine::root_stack_base_offset());
   __ PopRegister(FPREG);
   if (!FLAG_precompiled_mode) __ RestoreCodePointer();
   if (FLAG_precompiled_mode)  __ movq(PP, compiler::Address(THR, Thread::global_object_pool_offset()));
-  __ StoreFieldToOffset(SPREG, kCoroutine, Coroutine::stack_base_offset());
-
-  __ LeaveDartFrame();
-  __ set_constant_pool_allowed(true);
-
-  __ EnterFrame(0);
+
   __ PushObject(compiler::NullObject());
   __ CallRuntime(kExitCoroutineRuntimeEntry, 0);
   __ Drop(1);
-  __ LeaveFrame();
 }
 
 LocationSummary* CoroutineForkInstr::MakeLocationSummary(
@@ -8629,12 +8621,11 @@ void CoroutineForkInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
   __ StoreFieldToOffset(SPREG, kCallerCoroutine, Coroutine::stack_base_offset());
 
   __ LoadFieldFromOffset(SPREG, kForkedCoroutine, Coroutine::stack_base_offset());
-
   __ PushRegister(kForkedCoroutine);
-  compiler->EmitCallToStub(StubCode::CoroutineEntry());
+    compiler->GenerateStubCall(source(), StubCode::CoroutineEntry(), UntaggedPcDescriptors::kOther, locs(), deopt_id(), env());
   __ PopRegister(kForkedCoroutine);
-
   __ StoreFieldToOffset(SPREG, kForkedCoroutine, Coroutine::stack_base_offset());
+
   __ LoadCompressedFieldFromOffset(kCallerCoroutine, kForkedCoroutine, Coroutine::caller_offset());
 
   __ LoadFieldFromOffset(SPREG, kCallerCoroutine, Coroutine::stack_base_offset());
@@ -8665,7 +8656,7 @@ void CoroutineTransferInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
 
 #if defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64)
   SPILLS_LR_TO_FRAME({});
-#endif
+#endif  
   __ PushRegister(FPREG);
   __ StoreFieldToOffset(SPREG, kFromCoroutine, Coroutine::stack_base_offset());
 

runtime/vm/compiler/runtime_api.h

@@ -1038,6 +1038,8 @@ class Coroutine : public AllStatic {
  public:
   static word caller_offset();
   static word entry_offset();
+  static word root_stack_base_offset();
+  static word stack_root_offset();
   static word stack_base_offset();
   static word stack_limit_offset();
   static word InstanceSize();

runtime/vm/compiler/runtime_offsets_list.h

@@ -402,6 +402,8 @@
   FIELD(WeakReference, type_arguments_offset)                                  \
   FIELD(Coroutine, caller_offset)                                              \
   FIELD(Coroutine, entry_offset)                                               \
+  FIELD(Coroutine, root_stack_base_offset)                                     \
+  FIELD(Coroutine, stack_root_offset)                                          \
   FIELD(Coroutine, stack_base_offset)                                          \
   FIELD(Coroutine, stack_limit_offset)                                         \
   RANGE(Code, entry_point_offset, CodeEntryKind, CodeEntryKind::kNormal,       \

runtime/vm/constants_x64.h

@@ -394,16 +394,16 @@ struct CoroutineEntryABI {
 
 struct CoroutineInitializeABI {
   static constexpr Register kCoroutineReg = CoroutineEntryABI::kCoroutineReg;
-  static constexpr Register kSourceFrameReg = RBX;
-  static constexpr Register kStoreFrameReg = RCX;
-  static constexpr Register kSourceFrameSizeReg = RDX;
-  static constexpr Register kTempReg = TMP;
+  static constexpr Register kSourceFrameSizeReg = RAX;
+  static constexpr Register kSourceFrameReg = RCX;
 };
 
 struct CoroutineForkABI {
   static constexpr Register kCallerCoroutineReg = RSI;
   static constexpr Register kForkedCoroutineReg = CoroutineEntryABI::kCoroutineReg;
   static constexpr Register kStackLimitReg = RDX;
+  static constexpr Register kSourceFrameSizeReg = RAX;
+  static constexpr Register kSourceFrameReg = RCX;
 };
 
 struct CoroutineTransferABI {

runtime/vm/exceptions.cc

@@ -154,7 +154,6 @@ class ExceptionHandlerFinder : public StackResource {
                 (handler_pc !=
                  StubCode::AsyncExceptionHandler().EntryPoint())) {
               pc_ = frame->pc();
-              OS::Print("ExceptionHandlerFinder::find\n");
               code_ = &Code::Handle(frame->LookupDartCode());
               CatchEntryMovesRefPtr* cached_catch_entry_moves =
                   catch_entry_moves_cache_->Lookup(pc_);
@@ -689,7 +688,15 @@ NO_SANITIZE_SAFE_STACK  // This function manipulates the safestack pointer.
   typedef void (*ExcpHandler)(uword, uword, uword, Thread*);
   ExcpHandler func =
       reinterpret_cast<ExcpHandler>(StubCode::JumpToFrame().EntryPoint());
-
+  if (thread->has_coroutine()) {
+    CoroutinePtr found =
+        Coroutine::FindContainedCoroutine(thread->coroutine(), stack_pointer);
+    if (found.IsRawNull()) {
+      thread->ExitCoroutine();
+    } else {
+      thread->EnterCoroutine(found);
+    }
+  }
   if (thread->is_unwind_in_progress()) {
     thread->SetUnwindErrorInProgress(true);
   }

runtime/vm/object.cc

@@ -828,9 +828,7 @@ void Object::Init(IsolateGroup* isolate_group) {
   sentinel_class_ = cls.ptr();
 
   // Allocate and initialize the sentinel values.
-  {
-    *sentinel_ ^= Sentinel::New();
-  }
+  { *sentinel_ ^= Sentinel::New(); }
 
   // Allocate and initialize optimizing compiler constants.
   {
@@ -26629,19 +26627,40 @@ CodePtr SuspendState::GetCodeObject() const {
 #endif  // defined(DART_PRECOMPILED_RUNTIME)
 }
 
-CoroutinePtr Coroutine::New(void** stack_base, uintptr_t stack_size, FunctionPtr entry) {
-  const auto& coroutine = Coroutine::Handle(Object::Allocate<Coroutine>(Heap::kOld));
+CoroutinePtr Coroutine::New(void** stack_base,
+                            uintptr_t stack_size,
+                            FunctionPtr entry) {
+  const auto& coroutine =
+      Coroutine::Handle(Object::Allocate<Coroutine>(Heap::kOld));
+  coroutine.StoreNonPointer(&coroutine.untag()->root_stack_base_,
+                            (uword) nullptr);
+  coroutine.StoreNonPointer(&coroutine.untag()->stack_root_, (uword)stack_base);
   coroutine.StoreNonPointer(&coroutine.untag()->stack_base_, (uword)stack_base);
-  coroutine.StoreNonPointer(&coroutine.untag()->stack_limit_, (uword)stack_base - stack_size);
+  coroutine.StoreNonPointer(&coroutine.untag()->stack_limit_,
+                            (uword)stack_base - stack_size);
   coroutine.StoreCompressedPointer(&coroutine.untag()->entry_, entry);
-  coroutine.StoreCompressedPointer(&coroutine.untag()->caller_, coroutine.ptr());
+  coroutine.StoreCompressedPointer(&coroutine.untag()->caller_, null());
   return coroutine.ptr();
 }
 
 const char* Coroutine::ToCString() const {
   return "Coroutine";
 }
 
+CoroutinePtr Coroutine::FindContainedCoroutine(CoroutinePtr current,
+                                               uword stack_pointer) {
+  CoroutinePtr found = current;
+  IntMap<CoroutinePtr> processed;
+  while (!found.IsRawNull() && !processed.HasKey((uword)found->untag())) {
+    if (stack_pointer > found->untag()->stack_limit() && stack_pointer <= found->untag()->stack_root()) {
+      return found;
+    }
+    processed.Insert((uword)found->untag(), found);
+    found = found->untag()->caller();
+  }
+  return null();
+}
+
 void RegExp::set_pattern(const String& pattern) const {
   untag()->set_pattern(pattern.ptr());
 }
@@ -28026,4 +28045,3 @@ ArrayPtr RecordShape::GetFieldNames(Thread* thread) const {
 }
 
 }  // namespace dart
-

runtime/vm/object.h

@@ -12707,12 +12707,24 @@ class Coroutine : public Instance {
   }
   static CoroutinePtr New(void** stack_base, uintptr_t stack_size, FunctionPtr entry);
 
+  static CoroutinePtr FindContainedCoroutine(CoroutinePtr current, uword stack_pointer);
+
   CoroutinePtr caller() const { return untag()->caller(); }
   static uword caller_offset() { return OFFSET_OF(UntaggedCoroutine, caller_); }
 
   FunctionPtr entry() const { return untag()->entry(); }
   static uword entry_offset() { return OFFSET_OF(UntaggedCoroutine, entry_); }
 
+  uword root_stack_base() const { return untag()->root_stack_base_; }
+  static uword root_stack_base_offset() {
+    return OFFSET_OF(UntaggedCoroutine, root_stack_base_);
+  }
+
+  uword stack_root() const { return untag()->stack_root_; }
+  static uword stack_root_offset() {
+    return OFFSET_OF(UntaggedCoroutine, stack_root_);
+  }
+
   uword stack_base() const { return untag()->stack_base_; }
   static uword stack_base_offset() {
     return OFFSET_OF(UntaggedCoroutine, stack_base_);

runtime/vm/raw_object.h

@@ -3781,10 +3781,13 @@ class UntaggedCoroutine : public UntaggedInstance {
   COMPRESSED_POINTER_FIELD(FunctionPtr, entry)
   VISIT_TO(entry)
   CompressedObjectPtr* to_snapshot(Snapshot::Kind kind) { return to(); }
+  uword root_stack_base_;
+  uword stack_root_;
   uword stack_base_;
   uword stack_limit_;
 public:
   uword stack_base() const { return stack_base_; }
+  uword stack_root() const { return stack_root_; }
   uword stack_limit() const { return stack_limit_; }
 };
 

runtime/vm/raw_object_fields.cc

@@ -234,6 +234,8 @@ namespace dart {
   F(FutureOr, type_arguments_)                                                 \
   F(Coroutine, caller_)                                                        \
   F(Coroutine, entry_)                                                         \
+  F(Coroutine, root_stack_base_)                                               \
+  F(Coroutine, stack_root_)                                                    \
   F(Coroutine, stack_base_)                                                    \
   F(Coroutine, stack_limit_)
 

runtime/vm/runtime_entry.cc

@@ -69,7 +69,7 @@ DEFINE_FLAG(bool,
             false,
             "Ensure results of allocation via runtime calls are not in an "
             "active TLAB.");
-DEFINE_FLAG(bool, trace_deoptimization, false, "Trace deoptimization");
+DEFINE_FLAG(bool, trace_deoptimization, true, "Trace deoptimization");
 DEFINE_FLAG(bool,
             trace_deoptimization_verbose,
             false,

runtime/vm/stack_frame.cc

@@ -158,7 +158,6 @@ bool StackFrame::IsStubFrame() const {
   // where Thread::Current() is nullptr, so we cannot create a NoSafepointScope.
   NoSafepointScope no_safepoint;
 #endif
-
   CodePtr code = GetCodeObject();
   ASSERT(code != Object::null());
   auto const cid = Code::OwnerClassIdOf(code);
@@ -634,6 +633,7 @@ StackFrame* StackFrameIterator::FrameSetIterator::NextFrame(bool validate) {
     sp_ = frame->GetCallerSp();
     fp_ = frame->GetCallerFp();
     pc_ = frame->GetCallerPc();
+    Unpoison();
     return NextFrame(validate);
   }
   frame = &stack_frame_;

sdk/lib/_internal/vm/lib/fiber_patch.dart

@@ -26,6 +26,7 @@ class _Coroutine {
 @patch
 class Fiber {
   final _Coroutine _current;
+  final void Function() _entry;
 
   @patch
   FiberState get state => _state;
@@ -35,25 +36,36 @@ class Fiber {
   @pragma("vm:prefer-inline")
   Fiber._({required int size, required void Function() entry, required String name})
       : this.name = name,
-        _current = _Coroutine._(size, entry);
+        _entry = entry,
+        _current = _Coroutine._(size, _run);
 
   @patch
   @pragma("vm:prefer-inline")
   void start() {
+    Fiber._owner = this;
     _coroutineInitialize(_current);
   }
 
   @patch
   @pragma("vm:prefer-inline")
   void transfer(Fiber to) {
-     _coroutineTransfer(_current, to._current);
+    Fiber._owner = to;
+    to._caller = this;
+    _coroutineTransfer(_current, to._current);
   }
 
   @patch
   @pragma("vm:prefer-inline")
   void fork(Fiber to) {
+    Fiber._owner = to;
+    to._caller = this;
     _coroutineFork(_current, to._current);
   }
+
+  @pragma("vm:never-inline")
+  static void _run() {
+    Fiber._owner._entry();
+  }
 
   @patch
   static void idle() {

sdk/lib/fiber/fiber.dart

@@ -13,14 +13,27 @@ enum FiberState {
 
 class Fiber {
   final String name;
-  static late Fiber _main;
+  late Fiber _caller;
+  static late Fiber _owner;
 
   external static void idle();
+
+  @pragma("vm:prefer-inline")
+  static void spawn(Fiber to) {
+    _owner.fork(to);
+  }
+
+  @pragma("vm:prefer-inline")
+  static void suspend() {
+    _owner.transfer(_owner._caller);
+  }
 
   external FiberState get state;
   external Fiber._({required int size, required void Function() entry, required String name});
+
   factory Fiber.main({int size = _kDefaultStackSize, required void Function() entry}) => Fiber._(size: size, entry: entry, name: "main");
   factory Fiber.child({int size = _kDefaultStackSize, required void Function() entry, required String name}) => Fiber._(size: size, entry: entry, name: name);
+
   external void start();
   external void transfer(Fiber to);
   external void fork(Fiber to);

tests/lib/fiber/fiber_bench.dart

@@ -13,16 +13,16 @@ void main() {
 void mainEntry() {
   final sw = Stopwatch();
   sw.start();
-  mainFiber.fork(childFiber);
+  Fiber.spawn(childFiber);
   while (--value > 0) {
-    mainFiber.transfer(childFiber);
+    Fiber.suspend();
   }
   sw.stop();
-  print(sw.elapsedMicroseconds / counter);
+  print("Latency per switch: ${sw.elapsedMicroseconds / counter} [micros]");
 }
 
 void childEntry() {
   while (--value > 0) {
-    childFiber.transfer(mainFiber);
+    Fiber.suspend();
   }
 }