-
Notifications
You must be signed in to change notification settings - Fork 464
/
Copy pathevent_kevent.c
3304 lines (3006 loc) · 102 KB
/
event_kevent.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* Copyright (c) 2008-2016 Apple Inc. All rights reserved.
*
* @APPLE_APACHE_LICENSE_HEADER_START@
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* @APPLE_APACHE_LICENSE_HEADER_END@
*/
#include "internal.h"
#if DISPATCH_EVENT_BACKEND_KEVENT
#if HAVE_MACH
#include "protocol.h"
#include "protocolServer.h"
#endif
#if DISPATCH_USE_KEVENT_WORKQUEUE && !DISPATCH_USE_KEVENT_QOS
#error unsupported configuration
#endif
#define DISPATCH_KEVENT_MUXED_MARKER 1ul
#define DISPATCH_MACH_AUDIT_TOKEN_PID (5)
#define dispatch_kevent_udata_t __typeof__(((dispatch_kevent_t)NULL)->udata)
typedef struct dispatch_muxnote_s {
LIST_ENTRY(dispatch_muxnote_s) dmn_list;
LIST_HEAD(, dispatch_unote_linkage_s) dmn_unotes_head;
dispatch_kevent_s dmn_kev DISPATCH_ATOMIC64_ALIGN;
} *dispatch_muxnote_t;
LIST_HEAD(dispatch_muxnote_bucket_s, dispatch_muxnote_s);
DISPATCH_STATIC_GLOBAL(bool _dispatch_timers_force_max_leeway);
DISPATCH_STATIC_GLOBAL(dispatch_once_t _dispatch_kq_poll_pred);
DISPATCH_STATIC_GLOBAL(struct dispatch_muxnote_bucket_s _dispatch_sources[DSL_HASH_SIZE]);
#if defined(__APPLE__)
#define DISPATCH_NOTE_CLOCK_WALL NOTE_NSECONDS | NOTE_MACH_CONTINUOUS_TIME
#define DISPATCH_NOTE_CLOCK_MONOTONIC NOTE_MACHTIME | NOTE_MACH_CONTINUOUS_TIME
#define DISPATCH_NOTE_CLOCK_UPTIME NOTE_MACHTIME
#else
#define DISPATCH_NOTE_CLOCK_WALL 0
#define DISPATCH_NOTE_CLOCK_MONOTONIC 0
#define DISPATCH_NOTE_CLOCK_UPTIME 0
#endif
static const uint32_t _dispatch_timer_index_to_fflags[] = {
#define DISPATCH_TIMER_FFLAGS_INIT(kind, qos, note) \
[DISPATCH_TIMER_INDEX(DISPATCH_CLOCK_##kind, DISPATCH_TIMER_QOS_##qos)] = \
DISPATCH_NOTE_CLOCK_##kind | NOTE_ABSOLUTE | NOTE_LEEWAY | (note)
DISPATCH_TIMER_FFLAGS_INIT(WALL, NORMAL, 0),
DISPATCH_TIMER_FFLAGS_INIT(UPTIME, NORMAL, 0),
DISPATCH_TIMER_FFLAGS_INIT(MONOTONIC, NORMAL, 0),
#if DISPATCH_HAVE_TIMER_QOS
DISPATCH_TIMER_FFLAGS_INIT(WALL, CRITICAL, NOTE_CRITICAL),
DISPATCH_TIMER_FFLAGS_INIT(UPTIME, CRITICAL, NOTE_CRITICAL),
DISPATCH_TIMER_FFLAGS_INIT(MONOTONIC, CRITICAL, NOTE_CRITICAL),
DISPATCH_TIMER_FFLAGS_INIT(WALL, BACKGROUND, NOTE_BACKGROUND),
DISPATCH_TIMER_FFLAGS_INIT(UPTIME, BACKGROUND, NOTE_BACKGROUND),
DISPATCH_TIMER_FFLAGS_INIT(MONOTONIC, BACKGROUND, NOTE_BACKGROUND),
#endif
#undef DISPATCH_TIMER_FFLAGS_INIT
};
static inline void _dispatch_kevent_timer_drain(dispatch_kevent_t ke);
#if DISPATCH_USE_KEVENT_WORKLOOP
static void _dispatch_kevent_workloop_poke_drain(dispatch_kevent_t ke);
#endif
#pragma mark -
#pragma mark kevent debug
DISPATCH_NOINLINE
static const char *
_evfiltstr(short filt)
{
switch (filt) {
#define _evfilt2(f) case (f): return #f
_evfilt2(EVFILT_READ);
_evfilt2(EVFILT_WRITE);
_evfilt2(EVFILT_SIGNAL);
_evfilt2(EVFILT_TIMER);
#ifdef DISPATCH_EVENT_BACKEND_KEVENT
_evfilt2(EVFILT_AIO);
_evfilt2(EVFILT_VNODE);
_evfilt2(EVFILT_PROC);
#if HAVE_MACH
_evfilt2(EVFILT_MACHPORT);
_evfilt2(DISPATCH_EVFILT_MACH_NOTIFICATION);
#endif
_evfilt2(EVFILT_FS);
_evfilt2(EVFILT_USER);
#ifdef EVFILT_SOCK
_evfilt2(EVFILT_SOCK);
#endif
#ifdef EVFILT_MEMORYSTATUS
_evfilt2(EVFILT_MEMORYSTATUS);
#endif
#if DISPATCH_USE_KEVENT_WORKLOOP
_evfilt2(EVFILT_WORKLOOP);
#endif // DISPATCH_USE_KEVENT_WORKLOOP
#endif // DISPATCH_EVENT_BACKEND_KEVENT
_evfilt2(DISPATCH_EVFILT_TIMER);
_evfilt2(DISPATCH_EVFILT_TIMER_WITH_CLOCK);
_evfilt2(DISPATCH_EVFILT_CUSTOM_ADD);
_evfilt2(DISPATCH_EVFILT_CUSTOM_OR);
_evfilt2(DISPATCH_EVFILT_CUSTOM_REPLACE);
default:
return "EVFILT_missing";
}
}
#if DISPATCH_DEBUG
static const char *
_evflagstr2(uint16_t *flagsp)
{
#define _evflag2(f) \
if ((*flagsp & (f)) == (f) && (f)) { \
*flagsp &= ~(f); \
return #f "|"; \
}
_evflag2(EV_ADD);
_evflag2(EV_DELETE);
_evflag2(EV_ENABLE);
_evflag2(EV_DISABLE);
_evflag2(EV_ONESHOT);
_evflag2(EV_CLEAR);
_evflag2(EV_RECEIPT);
_evflag2(EV_DISPATCH);
_evflag2(EV_UDATA_SPECIFIC);
#ifdef EV_POLL
_evflag2(EV_POLL);
#endif
#ifdef EV_OOBAND
_evflag2(EV_OOBAND);
#endif
_evflag2(EV_ERROR);
_evflag2(EV_EOF);
_evflag2(EV_VANISHED);
*flagsp = 0;
return "EV_UNKNOWN ";
}
DISPATCH_NOINLINE
static const char *
_evflagstr(uint16_t flags, char *str, size_t strsize)
{
str[0] = 0;
while (flags) {
strlcat(str, _evflagstr2(&flags), strsize);
}
size_t sz = strlen(str);
if (sz) str[sz-1] = 0;
return str;
}
DISPATCH_NOINLINE
static void
dispatch_kevent_debug(const char *verb, const dispatch_kevent_s *kev,
int i, int n, const char *function, unsigned int line)
{
char flagstr[256];
char i_n[31];
if (n > 1) {
snprintf(i_n, sizeof(i_n), "%d/%d ", i + 1, n);
} else {
i_n[0] = '\0';
}
if (verb == NULL) {
if (kev->flags & EV_DELETE) {
verb = "deleting";
} else if (kev->flags & EV_ADD) {
verb = "adding";
} else {
verb = "updating";
}
}
#if DISPATCH_USE_KEVENT_QOS
_dispatch_debug("%s kevent[%p] %s= { ident = 0x%llx, filter = %s, "
"flags = %s (0x%x), fflags = 0x%x, data = 0x%llx, udata = 0x%llx, "
"qos = 0x%x, ext[0] = 0x%llx, ext[1] = 0x%llx, ext[2] = 0x%llx, "
"ext[3] = 0x%llx }: %s #%u", verb, kev, i_n,
(unsigned long long)kev->ident, _evfiltstr(kev->filter),
_evflagstr(kev->flags, flagstr, sizeof(flagstr)), kev->flags, kev->fflags,
(unsigned long long)kev->data, (unsigned long long)kev->udata, kev->qos,
kev->ext[0], kev->ext[1], kev->ext[2], kev->ext[3],
function, line);
#else
_dispatch_debug("%s kevent[%p] %s= { ident = 0x%llx, filter = %s, "
"flags = %s (0x%x), fflags = 0x%x, data = 0x%llx, udata = 0x%llx}: "
"%s #%u", verb, kev, i_n,
(unsigned long long)kev->ident, _evfiltstr(kev->filter),
_evflagstr(kev->flags, flagstr, sizeof(flagstr)), kev->flags,
kev->fflags, (unsigned long long)kev->data,
(unsigned long long)kev->udata,
function, line);
#endif
}
#else
static inline void
dispatch_kevent_debug(const char *verb, const dispatch_kevent_s *kev,
int i, int n, const char *function, unsigned int line)
{
(void)verb; (void)kev; (void)i; (void)n; (void)function; (void)line;
}
#endif // DISPATCH_DEBUG
#define _dispatch_kevent_debug_n(verb, _kev, i, n) \
dispatch_kevent_debug(verb, _kev, i, n, __FUNCTION__, __LINE__)
#define _dispatch_kevent_debug(verb, _kev) \
_dispatch_kevent_debug_n(verb, _kev, 0, 0)
#if DISPATCH_MGR_QUEUE_DEBUG
#define _dispatch_kevent_mgr_debug(verb, kev) _dispatch_kevent_debug(verb, kev)
#else
#define _dispatch_kevent_mgr_debug(verb, kev) ((void)verb, (void)kev)
#endif // DISPATCH_MGR_QUEUE_DEBUG
#if DISPATCH_WLH_DEBUG
#define _dispatch_kevent_wlh_debug(verb, kev) _dispatch_kevent_debug(verb, kev)
#else
#define _dispatch_kevent_wlh_debug(verb, kev) ((void)verb, (void)kev)
#endif // DISPATCH_WLH_DEBUG
#define _dispatch_du_debug(what, du) \
_dispatch_debug("kevent-source[%p]: %s kevent[%p] " \
"{ filter = %s, ident = 0x%x }", \
_dispatch_wref2ptr((du)->du_owner_wref), what, \
(du), _evfiltstr((du)->du_filter), (du)->du_ident)
#if DISPATCH_MACHPORT_DEBUG
#ifndef MACH_PORT_TYPE_SPREQUEST
#define MACH_PORT_TYPE_SPREQUEST 0x40000000
#endif
DISPATCH_NOINLINE
void
dispatch_debug_machport(mach_port_t name, const char* str)
{
mach_port_type_t type;
mach_msg_bits_t ns = 0, nr = 0, nso = 0, nd = 0;
unsigned int dnreqs = 0, dnrsiz;
kern_return_t kr = mach_port_type(mach_task_self(), name, &type);
if (kr) {
_dispatch_log("machport[0x%08x] = { error(0x%x) \"%s\" }: %s", name,
kr, mach_error_string(kr), str);
return;
}
if (type & MACH_PORT_TYPE_SEND) {
(void)dispatch_assume_zero(mach_port_get_refs(mach_task_self(), name,
MACH_PORT_RIGHT_SEND, &ns));
}
if (type & MACH_PORT_TYPE_SEND_ONCE) {
(void)dispatch_assume_zero(mach_port_get_refs(mach_task_self(), name,
MACH_PORT_RIGHT_SEND_ONCE, &nso));
}
if (type & MACH_PORT_TYPE_DEAD_NAME) {
(void)dispatch_assume_zero(mach_port_get_refs(mach_task_self(), name,
MACH_PORT_RIGHT_DEAD_NAME, &nd));
}
if (type & (MACH_PORT_TYPE_RECEIVE|MACH_PORT_TYPE_SEND)) {
kr = mach_port_dnrequest_info(mach_task_self(), name, &dnrsiz, &dnreqs);
if (kr != KERN_INVALID_RIGHT) (void)dispatch_assume_zero(kr);
}
if (type & MACH_PORT_TYPE_RECEIVE) {
mach_port_status_t status = { .mps_pset = 0, };
mach_msg_type_number_t cnt = MACH_PORT_RECEIVE_STATUS_COUNT;
(void)dispatch_assume_zero(mach_port_get_refs(mach_task_self(), name,
MACH_PORT_RIGHT_RECEIVE, &nr));
(void)dispatch_assume_zero(mach_port_get_attributes(mach_task_self(),
name, MACH_PORT_RECEIVE_STATUS, (void*)&status, &cnt));
_dispatch_log("machport[0x%08x] = { R(%03u) S(%03u) SO(%03u) D(%03u) "
"dnreqs(%03u) spreq(%s) nsreq(%s) pdreq(%s) srights(%s) "
"sorights(%03u) qlim(%03u) msgcount(%03u) mkscount(%03u) "
"seqno(%03u) }: %s", name, nr, ns, nso, nd, dnreqs,
type & MACH_PORT_TYPE_SPREQUEST ? "Y":"N",
status.mps_nsrequest ? "Y":"N", status.mps_pdrequest ? "Y":"N",
status.mps_srights ? "Y":"N", status.mps_sorights,
status.mps_qlimit, status.mps_msgcount, status.mps_mscount,
status.mps_seqno, str);
} else if (type & (MACH_PORT_TYPE_SEND|MACH_PORT_TYPE_SEND_ONCE|
MACH_PORT_TYPE_DEAD_NAME)) {
_dispatch_log("machport[0x%08x] = { R(%03u) S(%03u) SO(%03u) D(%03u) "
"dnreqs(%03u) spreq(%s) }: %s", name, nr, ns, nso, nd, dnreqs,
type & MACH_PORT_TYPE_SPREQUEST ? "Y":"N", str);
} else {
_dispatch_log("machport[0x%08x] = { type(0x%08x) }: %s", name, type,
str);
}
}
#endif
#pragma mark dispatch_kevent_t
#if HAVE_MACH
DISPATCH_STATIC_GLOBAL(dispatch_once_t _dispatch_mach_host_port_pred);
DISPATCH_STATIC_GLOBAL(mach_port_t _dispatch_mach_host_port);
static inline void*
_dispatch_kevent_mach_msg_buf(dispatch_kevent_t ke)
{
return (void*)ke->ext[0];
}
static inline mach_msg_size_t
_dispatch_kevent_mach_msg_size(dispatch_kevent_t ke)
{
// buffer size in the successful receive case, but message size (like
// msgh_size) in the MACH_RCV_TOO_LARGE case, i.e. add trailer size.
return (mach_msg_size_t)ke->ext[1];
}
static void _dispatch_kevent_mach_msg_drain(dispatch_kevent_t ke);
static inline void _dispatch_mach_host_calendar_change_register(void);
// DISPATCH_MACH_NOTIFICATION_ARMED are muxnotes that aren't registered with
// kevent for real, but with mach_port_request_notification()
//
// the kevent structure is used for bookkeeping:
// - ident, filter, flags and fflags have their usual meaning
// - data is used to monitor the actual state of the
// mach_port_request_notification()
// - ext[0] is a boolean that trackes whether the notification is armed or not
#define DISPATCH_MACH_NOTIFICATION_ARMED(dmn) ((dmn)->dmn_kev.ext[0])
#endif
DISPATCH_ALWAYS_INLINE
static dispatch_muxnote_t
_dispatch_kevent_get_muxnote(dispatch_kevent_t ke)
{
uintptr_t dmn_addr = (uintptr_t)ke->udata & ~DISPATCH_KEVENT_MUXED_MARKER;
return (dispatch_muxnote_t)dmn_addr;
}
DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_kevent_unote_is_muxed(dispatch_kevent_t ke)
{
return ((uintptr_t)ke->udata) & DISPATCH_KEVENT_MUXED_MARKER;
}
DISPATCH_ALWAYS_INLINE
static dispatch_unote_t
_dispatch_kevent_get_unote(dispatch_kevent_t ke)
{
dispatch_assert(_dispatch_kevent_unote_is_muxed(ke) == false);
return (dispatch_unote_t){ ._du = (dispatch_unote_class_t)ke->udata };
}
DISPATCH_NOINLINE
static void
_dispatch_kevent_print_error(dispatch_kevent_t ke)
{
dispatch_unote_class_t du = NULL;
_dispatch_debug("kevent[0x%llx]: handling error",
(unsigned long long)ke->udata);
if (ke->flags & EV_DELETE) {
if (ke->flags & EV_UDATA_SPECIFIC) {
if (ke->data == EINPROGRESS) {
// deferred EV_DELETE
return;
}
}
// for EV_DELETE if the update was deferred we may have reclaimed
// the udata already, and it is unsafe to dereference it now.
} else if (_dispatch_kevent_unote_is_muxed(ke)) {
ke->flags |= _dispatch_kevent_get_muxnote(ke)->dmn_kev.flags;
} else if (ke->udata) {
du = (dispatch_unote_class_t)(uintptr_t)ke->udata;
if (!_dispatch_unote_registered(du)) {
ke->flags |= EV_ADD;
}
}
switch (ke->data) {
case 0:
return;
case ERANGE: /* A broken QoS was passed to kevent_id() */
DISPATCH_INTERNAL_CRASH(ke->qos, "Invalid kevent priority");
default:
// log the unexpected error
_dispatch_bug_kevent_client("kevent", _evfiltstr(ke->filter),
!ke->udata ? NULL :
ke->flags & EV_DELETE ? "delete" :
ke->flags & EV_ADD ? "add" :
ke->flags & EV_ENABLE ? "enable" : "monitor",
(int)ke->data, ke->ident, ke->udata, du);
}
}
DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_kevent_merge_ev_flags(dispatch_unote_t du, uint32_t flags)
{
if (unlikely(!(flags & EV_UDATA_SPECIFIC) && (flags & EV_ONESHOT))) {
_dispatch_unote_unregister(du, DUU_DELETE_ACK | DUU_MUST_SUCCEED);
return;
}
if (flags & EV_DELETE) {
// When a speculative deletion is requested by libdispatch,
// and the kernel is about to deliver an event, it can acknowledge
// our wish by delivering the event as a (EV_DELETE | EV_ONESHOT)
// event and dropping the knote at once.
_dispatch_unote_state_set(du, DU_STATE_NEEDS_DELETE);
} else if (flags & (EV_ONESHOT | EV_VANISHED)) {
// EV_VANISHED events if re-enabled will produce another EV_VANISHED
// event. To avoid an infinite loop of such events, mark the unote
// as needing deletion so that _dispatch_unote_needs_rearm()
// eventually returns false.
//
// mach channels crash on EV_VANISHED, and dispatch sources stay
// in a limbo until canceled (explicitly or not).
dispatch_unote_state_t du_state = _dispatch_unote_state(du);
du_state |= DU_STATE_NEEDS_DELETE;
du_state &= ~DU_STATE_ARMED;
_dispatch_unote_state_set(du, du_state);
} else if (likely(flags & EV_DISPATCH)) {
_dispatch_unote_state_clear_bit(du, DU_STATE_ARMED);
} else {
return;
}
_dispatch_du_debug((flags & EV_VANISHED) ? "vanished" :
(flags & EV_DELETE) ? "deleted oneshot" :
(flags & EV_ONESHOT) ? "oneshot" : "disarmed", du._du);
}
DISPATCH_NOINLINE
static void
_dispatch_kevent_merge(dispatch_unote_t du, dispatch_kevent_t ke)
{
dispatch_unote_action_t action = dux_type(du._du)->dst_action;
pthread_priority_t pp = 0;
uintptr_t data;
// once we modify the queue atomic flags below, it will allow concurrent
// threads running _dispatch_source_invoke2 to dispose of the source,
// so we can't safely borrow the reference we get from the muxnote udata
// anymore, and need our own <rdar://20382435>
_dispatch_retain_unote_owner(du);
switch (action) {
case DISPATCH_UNOTE_ACTION_PASS_DATA:
data = (uintptr_t)ke->data;
break;
case DISPATCH_UNOTE_ACTION_PASS_FFLAGS:
data = (uintptr_t)ke->fflags;
#if HAVE_MACH
if (du._du->du_filter == EVFILT_MACHPORT) {
data = DISPATCH_MACH_RECV_MESSAGE;
}
#endif
break;
case DISPATCH_UNOTE_ACTION_SOURCE_SET_DATA:
// ke->data is signed and "negative available data" makes no sense
// zero bytes happens when EV_EOF is set
dispatch_assert(ke->data >= 0l);
data = (unsigned long)ke->data;
os_atomic_store2o(du._dr, ds_pending_data, ~data, relaxed);
break;
case DISPATCH_UNOTE_ACTION_SOURCE_ADD_DATA:
data = (unsigned long)ke->data;
if (data) os_atomic_add2o(du._dr, ds_pending_data, data, relaxed);
break;
case DISPATCH_UNOTE_ACTION_SOURCE_OR_FFLAGS:
data = ke->fflags & du._du->du_fflags;
if (du._dr->du_has_extended_status) {
uint64_t odata, ndata, value;
uint32_t status = (uint32_t)ke->data;
// We combine the data and status into a single 64-bit value.
value = DISPATCH_SOURCE_COMBINE_DATA_AND_STATUS(data, status);
os_atomic_rmw_loop2o(du._dr, ds_pending_data, odata, ndata, relaxed, {
ndata = DISPATCH_SOURCE_GET_DATA(odata) | value;
});
#if HAVE_MACH
} else if (du._du->du_filter == EVFILT_MACHPORT) {
data = DISPATCH_MACH_RECV_MESSAGE;
os_atomic_store2o(du._dr, ds_pending_data, data, relaxed);
#endif
} else {
if (data) os_atomic_or2o(du._dr, ds_pending_data, data, relaxed);
}
break;
default:
DISPATCH_INTERNAL_CRASH(action, "Corrupt unote action");
}
_dispatch_kevent_merge_ev_flags(du, ke->flags);
#if DISPATCH_USE_KEVENT_QOS
pp = ((pthread_priority_t)ke->qos) & ~_PTHREAD_PRIORITY_FLAGS_MASK;
#endif
return dux_merge_evt(du._du, ke->flags, data, pp);
}
DISPATCH_NOINLINE
static void
_dispatch_kevent_merge_muxed(dispatch_kevent_t ke)
{
dispatch_muxnote_t dmn = _dispatch_kevent_get_muxnote(ke);
dispatch_unote_linkage_t dul, dul_next;
if (ke->flags & (EV_ONESHOT | EV_DELETE)) {
// tell _dispatch_unote_unregister_muxed() the kernel half is gone
dmn->dmn_kev.flags |= EV_DELETE;
}
LIST_FOREACH_SAFE(dul, &dmn->dmn_unotes_head, du_link, dul_next) {
_dispatch_kevent_merge(_dispatch_unote_linkage_get_unote(dul), ke);
}
}
DISPATCH_NOINLINE
static void
_dispatch_kevent_drain(dispatch_kevent_t ke)
{
if (ke->filter == EVFILT_USER) {
_dispatch_kevent_mgr_debug("received", ke);
return;
}
#if DISPATCH_USE_KEVENT_WORKLOOP
if (ke->filter == EVFILT_WORKLOOP) {
return _dispatch_kevent_workloop_poke_drain(ke);
}
#endif // DISPATCH_USE_KEVENT_WORKLOOP
_dispatch_kevent_debug("received", ke);
if (unlikely(ke->flags & EV_ERROR)) {
if (ke->filter == EVFILT_PROC && ke->data == ESRCH) {
// <rdar://problem/5067725&6626350> EVFILT_PROC may fail with ESRCH
// when the process exists but is a zombie. As a workaround, we
// simulate an exit event for any EVFILT_PROC with an invalid pid.
ke->flags = EV_UDATA_SPECIFIC | EV_ONESHOT | EV_DELETE;
ke->fflags = NOTE_EXIT;
ke->data = 0;
_dispatch_kevent_debug("synthetic NOTE_EXIT", ke);
} else {
return _dispatch_kevent_print_error(ke);
}
}
if (ke->filter == EVFILT_TIMER) {
return _dispatch_kevent_timer_drain(ke);
}
#if HAVE_MACH
if (ke->filter == EVFILT_MACHPORT && _dispatch_kevent_mach_msg_size(ke)) {
return _dispatch_kevent_mach_msg_drain(ke);
}
#endif
if (_dispatch_kevent_unote_is_muxed(ke)) {
return _dispatch_kevent_merge_muxed(ke);
}
return _dispatch_kevent_merge(_dispatch_kevent_get_unote(ke), ke);
}
#pragma mark dispatch_kq
#if DISPATCH_USE_MGR_THREAD
DISPATCH_NOINLINE
static void
_dispatch_kq_create(intptr_t *fd_ptr)
{
static const dispatch_kevent_s kev = {
.ident = 1,
.filter = EVFILT_USER,
.flags = EV_ADD|EV_CLEAR,
.udata = (dispatch_kevent_udata_t)DISPATCH_WLH_MANAGER,
};
int kqfd;
_dispatch_fork_becomes_unsafe();
#if DISPATCH_USE_GUARDED_FD
guardid_t guard = (uintptr_t)fd_ptr;
kqfd = guarded_kqueue_np(&guard, GUARD_CLOSE | GUARD_DUP);
#else
(void)guard_ptr;
kqfd = kqueue();
#endif
if (kqfd == -1) {
int err = errno;
switch (err) {
case EMFILE:
DISPATCH_CLIENT_CRASH(err, "kqueue() failure: "
"process is out of file descriptors");
break;
case ENFILE:
DISPATCH_CLIENT_CRASH(err, "kqueue() failure: "
"system is out of file descriptors");
break;
case ENOMEM:
DISPATCH_CLIENT_CRASH(err, "kqueue() failure: "
"kernel is out of memory");
break;
default:
DISPATCH_INTERNAL_CRASH(err, "kqueue() failure");
break;
}
}
#if DISPATCH_USE_KEVENT_QOS
dispatch_assume_zero(kevent_qos(kqfd, &kev, 1, NULL, 0, NULL, NULL, 0));
#else
dispatch_assume_zero(kevent(kqfd, &kev, 1, NULL, 0, NULL));
#endif
*fd_ptr = kqfd;
}
#endif
static inline int
_dispatch_kq_fd(void)
{
return (int)(intptr_t)_dispatch_mgr_q.do_ctxt;
}
static void
_dispatch_kq_init(void *context)
{
bool *kq_initialized = context;
_dispatch_fork_becomes_unsafe();
if (unlikely(getenv("LIBDISPATCH_TIMERS_FORCE_MAX_LEEWAY"))) {
_dispatch_timers_force_max_leeway = true;
}
*kq_initialized = true;
#if DISPATCH_USE_KEVENT_WORKQUEUE
_dispatch_kevent_workqueue_init();
if (_dispatch_kevent_workqueue_enabled) {
int r;
int kqfd = _dispatch_kq_fd();
const dispatch_kevent_s ke = {
.ident = 1,
.filter = EVFILT_USER,
.flags = EV_ADD|EV_CLEAR,
.qos = _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG,
.udata = (dispatch_kevent_udata_t)DISPATCH_WLH_MANAGER,
};
retry:
r = kevent_qos(kqfd, &ke, 1, NULL, 0, NULL, NULL,
KEVENT_FLAG_WORKQ|KEVENT_FLAG_IMMEDIATE);
if (unlikely(r == -1)) {
int err = errno;
switch (err) {
case EINTR:
goto retry;
default:
DISPATCH_CLIENT_CRASH(err,
"Failed to initalize workqueue kevent");
break;
}
}
return;
}
#endif // DISPATCH_USE_KEVENT_WORKQUEUE
#if DISPATCH_USE_MGR_THREAD
_dispatch_kq_create((intptr_t *)&_dispatch_mgr_q.do_ctxt);
_dispatch_trace_item_push(_dispatch_mgr_q.do_targetq, &_dispatch_mgr_q);
dx_push(_dispatch_mgr_q.do_targetq, &_dispatch_mgr_q, 0);
#endif // DISPATCH_USE_MGR_THREAD
}
#if DISPATCH_USE_MEMORYPRESSURE_SOURCE
static void _dispatch_memorypressure_init(void);
#else
#define _dispatch_memorypressure_init() ((void)0)
#endif
DISPATCH_NOINLINE
static int
_dispatch_kq_poll(dispatch_wlh_t wlh, dispatch_kevent_t ke, int n,
dispatch_kevent_t ke_out, int n_out, void *buf, size_t *avail,
uint32_t flags)
{
bool kq_initialized = false;
int r = 0;
dispatch_once_f(&_dispatch_kq_poll_pred, &kq_initialized, _dispatch_kq_init);
if (unlikely(kq_initialized)) {
// The calling thread was the one doing the initialization
//
// The event loop needs the memory pressure source and debug channel,
// however creating these will recursively call _dispatch_kq_poll(),
// so we can't quite initialize them under the dispatch once.
_dispatch_memorypressure_init();
_voucher_activity_debug_channel_init();
}
#if !DISPATCH_USE_KEVENT_QOS
if (flags & KEVENT_FLAG_ERROR_EVENTS) {
// emulate KEVENT_FLAG_ERROR_EVENTS
for (r = 0; r < n; r++) {
ke[r].flags |= EV_RECEIPT;
}
n_out = n;
}
#endif
retry:
if (unlikely(wlh == NULL)) {
DISPATCH_INTERNAL_CRASH(wlh, "Invalid wlh");
} else if (wlh == DISPATCH_WLH_ANON) {
int kqfd = _dispatch_kq_fd();
#if DISPATCH_USE_KEVENT_QOS
if (_dispatch_kevent_workqueue_enabled) {
flags |= KEVENT_FLAG_WORKQ;
}
r = kevent_qos(kqfd, ke, n, ke_out, n_out, buf, avail, flags);
#else
(void)buf;
(void)avail;
const struct timespec timeout_immediately = {}, *timeout = NULL;
if (flags & KEVENT_FLAG_IMMEDIATE) timeout = &timeout_immediately;
r = kevent(kqfd, ke, n, ke_out, n_out, timeout);
#endif
#if DISPATCH_USE_KEVENT_WORKLOOP
} else {
flags |= KEVENT_FLAG_WORKLOOP;
if (!(flags & KEVENT_FLAG_ERROR_EVENTS)) {
flags |= KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST;
}
r = kevent_id((uintptr_t)wlh, ke, n, ke_out, n_out, buf, avail, flags);
#endif // DISPATCH_USE_KEVENT_WORKLOOP
}
if (unlikely(r == -1)) {
int err = errno;
switch (err) {
case ENOMEM:
_dispatch_temporary_resource_shortage();
/* FALLTHROUGH */
case EINTR:
goto retry;
case EBADF:
DISPATCH_CLIENT_CRASH(err, "Do not close random Unix descriptors");
#if DISPATCH_USE_KEVENT_WORKLOOP
case ENOENT:
if ((flags & KEVENT_FLAG_ERROR_EVENTS) &&
(flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST)) {
return 0;
}
/* FALLTHROUGH */
#endif // DISPATCH_USE_KEVENT_WORKLOOP
default:
DISPATCH_CLIENT_CRASH(err, "Unexpected error from kevent");
}
}
return r;
}
DISPATCH_NOINLINE
static int
_dispatch_kq_drain(dispatch_wlh_t wlh, dispatch_kevent_t ke, int n,
uint32_t flags)
{
dispatch_kevent_s ke_out[DISPATCH_DEFERRED_ITEMS_EVENT_COUNT];
bool poll_for_events = !(flags & KEVENT_FLAG_ERROR_EVENTS);
int i, n_out = countof(ke_out), r = 0;
size_t *avail = NULL;
void *buf = NULL;
#if DISPATCH_USE_KEVENT_QOS
size_t size;
if (poll_for_events) {
size = DISPATCH_MACH_RECEIVE_MAX_INLINE_MESSAGE_SIZE +
DISPATCH_MACH_TRAILER_SIZE;
buf = alloca(size);
avail = &size;
}
#endif
#if DISPATCH_DEBUG
for (r = 0; r < n; r++) {
if (ke[r].filter != EVFILT_USER || DISPATCH_MGR_QUEUE_DEBUG) {
_dispatch_kevent_debug_n(NULL, ke + r, r, n);
}
}
#endif
if (poll_for_events) _dispatch_clear_return_to_kernel();
n = _dispatch_kq_poll(wlh, ke, n, ke_out, n_out, buf, avail, flags);
if (n == 0) {
r = 0;
} else if (flags & KEVENT_FLAG_ERROR_EVENTS) {
for (i = 0, r = 0; i < n; i++) {
if ((ke_out[i].flags & EV_ERROR) && ke_out[i].data) {
_dispatch_kevent_drain(&ke_out[i]);
r = (int)ke_out[i].data;
}
}
} else {
#if DISPATCH_USE_KEVENT_WORKLOOP
if (ke_out[0].flags & EV_ERROR) {
// When kevent returns errors it doesn't process the kqueue
// and doesn't rearm the return-to-kernel notification
// We need to assume we have to go back.
_dispatch_set_return_to_kernel();
}
#endif // DISPATCH_USE_KEVENT_WORKLOOP
for (i = 0, r = 0; i < n; i++) {
_dispatch_kevent_drain(&ke_out[i]);
}
}
return r;
}
DISPATCH_ALWAYS_INLINE
static inline int
_dispatch_kq_update_one(dispatch_wlh_t wlh, dispatch_kevent_t ke)
{
return _dispatch_kq_drain(wlh, ke, 1,
KEVENT_FLAG_IMMEDIATE | KEVENT_FLAG_ERROR_EVENTS);
}
DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_kq_update_all(dispatch_wlh_t wlh, dispatch_kevent_t ke, int n)
{
(void)_dispatch_kq_drain(wlh, ke, n,
KEVENT_FLAG_IMMEDIATE | KEVENT_FLAG_ERROR_EVENTS);
}
DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_kq_unote_set_kevent(dispatch_unote_t _du, dispatch_kevent_t dk,
uint16_t action)
{
dispatch_unote_class_t du = _du._du;
dispatch_source_type_t dst = dux_type(du);
uint16_t flags = dst->dst_flags | action;
if ((flags & EV_VANISHED) && !(flags & EV_ADD)) {
flags &= ~EV_VANISHED;
}
*dk = (dispatch_kevent_s){
.ident = du->du_ident,
.filter = dst->dst_filter,
.flags = flags,
.udata = (dispatch_kevent_udata_t)du,
.fflags = du->du_fflags | dst->dst_fflags,
.data = (__typeof__(dk->data))dst->dst_data,
#if DISPATCH_USE_KEVENT_QOS
.qos = (__typeof__(dk->qos))_dispatch_priority_to_pp_prefer_fallback(
du->du_priority),
#endif
};
(void)pp; // if DISPATCH_USE_KEVENT_QOS == 0
}
DISPATCH_ALWAYS_INLINE
static inline int
_dispatch_kq_deferred_find_slot(dispatch_deferred_items_t ddi,
int16_t filter, uint64_t ident, dispatch_kevent_udata_t udata)
{
dispatch_kevent_t events = ddi->ddi_eventlist;
int i;
for (i = 0; i < ddi->ddi_nevents; i++) {
if (events[i].filter == filter && events[i].ident == ident &&
events[i].udata == udata) {
break;
}
}
return i;
}
DISPATCH_ALWAYS_INLINE
static inline dispatch_kevent_t
_dispatch_kq_deferred_reuse_slot(dispatch_wlh_t wlh,
dispatch_deferred_items_t ddi, int slot)
{
if (wlh != DISPATCH_WLH_ANON) _dispatch_set_return_to_kernel();
if (unlikely(slot == ddi->ddi_maxevents)) {
int nevents = ddi->ddi_nevents;
ddi->ddi_nevents = 1;
_dispatch_kq_update_all(wlh, ddi->ddi_eventlist, nevents);
dispatch_assert(ddi->ddi_nevents == 1);
slot = 0;
} else if (slot == ddi->ddi_nevents) {
ddi->ddi_nevents++;
}
return ddi->ddi_eventlist + slot;
}
DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_kq_deferred_discard_slot(dispatch_deferred_items_t ddi, int slot)
{
if (slot < ddi->ddi_nevents) {
int last = --ddi->ddi_nevents;
if (slot != last) {
ddi->ddi_eventlist[slot] = ddi->ddi_eventlist[last];
}
}
}
DISPATCH_NOINLINE
static void
_dispatch_kq_deferred_update(dispatch_wlh_t wlh, dispatch_kevent_t ke)
{
dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
if (ddi && ddi->ddi_wlh == wlh && ddi->ddi_maxevents) {
int slot = _dispatch_kq_deferred_find_slot(ddi, ke->filter, ke->ident,
ke->udata);
dispatch_kevent_t dk = _dispatch_kq_deferred_reuse_slot(wlh, ddi, slot);
*dk = *ke;
if (ke->filter != EVFILT_USER) {
_dispatch_kevent_mgr_debug("deferred", ke);
}
} else {
_dispatch_kq_update_one(wlh, ke);
}
}
DISPATCH_NOINLINE
static int
_dispatch_kq_immediate_update(dispatch_wlh_t wlh, dispatch_kevent_t ke)
{
dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
if (ddi && ddi->ddi_wlh == wlh) {
int slot = _dispatch_kq_deferred_find_slot(ddi, ke->filter, ke->ident,
ke->udata);
_dispatch_kq_deferred_discard_slot(ddi, slot);
}
return _dispatch_kq_update_one(wlh, ke);
}
#if HAVE_MACH
void
_dispatch_sync_ipc_handoff_begin(dispatch_wlh_t wlh, mach_port_t port,
uint64_t _Atomic *addr)
{
#ifdef NOTE_WL_SYNC_IPC
dispatch_kevent_s ke = {
.ident = port,
.filter = EVFILT_WORKLOOP,
.flags = EV_ADD | EV_DISABLE,
.fflags = NOTE_WL_SYNC_IPC | NOTE_WL_IGNORE_ESTALE,
.udata = (uintptr_t)wlh,
.ext[EV_EXTIDX_WL_ADDR] = (uintptr_t)addr,
.ext[EV_EXTIDX_WL_MASK] = ~(uintptr_t)0,
.ext[EV_EXTIDX_WL_VALUE] = (uintptr_t)wlh,
};
int rc = _dispatch_kq_immediate_update(wlh, &ke);
if (unlikely(rc)) {
DISPATCH_INTERNAL_CRASH(rc, "Unexpected error from kevent");
}
#else
(void)wlh; (void)port; (void)addr;
#endif
}
void
_dispatch_sync_ipc_handoff_end(dispatch_wlh_t wlh, mach_port_t port)
{
#ifdef NOTE_WL_SYNC_IPC
dispatch_kevent_s ke = {
.ident = port,
.filter = EVFILT_WORKLOOP,
.flags = EV_ADD | EV_DELETE | EV_ENABLE,
.fflags = NOTE_WL_SYNC_IPC,
.udata = (uintptr_t)wlh,
};
_dispatch_kq_deferred_update(wlh, &ke);
#else
(void)wlh; (void)port;
#endif // NOTE_WL_SYNC_IPC
}
#endif
DISPATCH_NOINLINE
static bool
_dispatch_kq_unote_update(dispatch_wlh_t wlh, dispatch_unote_t _du,
uint16_t action_flags)
{
dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
dispatch_unote_class_t du = _du._du;
dispatch_kevent_t ke;
int r = 0;
if (action_flags & EV_ADD) {
// as soon as we register we may get an event delivery and it has to
// see du_state already set, else it will not unregister the kevent